2024-11-19 17:47:55,544 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 17:47:55,555 main DEBUG Took 0.009553 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 17:47:55,556 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 17:47:55,556 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 17:47:55,557 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 17:47:55,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,566 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 17:47:55,578 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,580 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,580 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,581 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,581 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,583 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,583 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,585 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,586 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,586 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,587 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,588 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,589 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,589 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:55,590 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,590 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 17:47:55,592 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:55,593 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 17:47:55,595 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 17:47:55,595 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 17:47:55,596 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 17:47:55,597 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 17:47:55,605 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 17:47:55,607 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 17:47:55,609 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 17:47:55,609 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 17:47:55,610 main DEBUG createAppenders(={Console}) 2024-11-19 17:47:55,611 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 17:47:55,611 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 17:47:55,611 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 17:47:55,612 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 17:47:55,612 main DEBUG OutputStream closed 2024-11-19 17:47:55,612 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 17:47:55,612 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 17:47:55,613 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 17:47:55,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 17:47:55,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 17:47:55,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 17:47:55,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 17:47:55,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 17:47:55,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 17:47:55,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 17:47:55,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 17:47:55,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 17:47:55,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 17:47:55,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 17:47:55,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 17:47:55,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 17:47:55,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 17:47:55,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 17:47:55,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 17:47:55,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 17:47:55,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 17:47:55,690 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 17:47:55,691 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 17:47:55,691 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 17:47:55,692 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T17:47:55,899 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4 2024-11-19 17:47:55,903 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 17:47:55,904 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T17:47:55,913 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-19T17:47:55,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T17:47:55,966 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19, deleteOnExit=true 2024-11-19T17:47:55,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T17:47:55,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/test.cache.data in system properties and HBase conf 2024-11-19T17:47:55,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T17:47:55,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/hadoop.log.dir in system properties and HBase conf 2024-11-19T17:47:55,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T17:47:55,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T17:47:55,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T17:47:56,087 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T17:47:56,217 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:56,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T17:47:56,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T17:47:56,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T17:47:56,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T17:47:56,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T17:47:56,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T17:47:56,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T17:47:56,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T17:47:56,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T17:47:56,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/nfs.dump.dir in system properties and HBase conf 2024-11-19T17:47:56,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/java.io.tmpdir in system properties and HBase conf 2024-11-19T17:47:56,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T17:47:56,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T17:47:56,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T17:47:57,428 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T17:47:57,505 INFO [Time-limited test {}] log.Log(170): Logging initialized @2860ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T17:47:57,574 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T17:47:57,634 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T17:47:57,653 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T17:47:57,654 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T17:47:57,655 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T17:47:57,666 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T17:47:57,669 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/hadoop.log.dir/,AVAILABLE} 2024-11-19T17:47:57,669 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T17:47:57,848 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/java.io.tmpdir/jetty-localhost-41273-hadoop-hdfs-3_4_1-tests_jar-_-any-5797053895548557432/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T17:47:57,859 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41273} 2024-11-19T17:47:57,859 INFO [Time-limited test {}] server.Server(415): Started @3215ms 2024-11-19T17:47:58,427 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T17:47:58,435 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T17:47:58,436 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T17:47:58,437 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T17:47:58,437 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T17:47:58,438 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/hadoop.log.dir/,AVAILABLE} 2024-11-19T17:47:58,438 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T17:47:58,541 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/java.io.tmpdir/jetty-localhost-37309-hadoop-hdfs-3_4_1-tests_jar-_-any-2128736669309941371/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T17:47:58,542 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:37309} 2024-11-19T17:47:58,542 INFO [Time-limited test {}] server.Server(415): Started @3898ms 2024-11-19T17:47:58,590 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T17:47:59,567 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/data/data2/current/BP-1826768862-172.17.0.2-1732038476880/current, will proceed with Du for space computation calculation, 2024-11-19T17:47:59,567 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/data/data1/current/BP-1826768862-172.17.0.2-1732038476880/current, will proceed with Du for space computation calculation, 2024-11-19T17:47:59,599 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T17:47:59,642 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x690d07acffc94ce3 with lease ID 0xa1c0cea54474c8b: Processing first storage report for DS-e4ab4247-02c2-4707-8ea9-21a06b1ee73a from datanode DatanodeRegistration(127.0.0.1:45951, datanodeUuid=b611bc95-043b-4517-91a5-22a5c778988f, infoPort=43961, infoSecurePort=0, ipcPort=35285, storageInfo=lv=-57;cid=testClusterID;nsid=342725190;c=1732038476880) 2024-11-19T17:47:59,644 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x690d07acffc94ce3 with lease ID 0xa1c0cea54474c8b: from storage DS-e4ab4247-02c2-4707-8ea9-21a06b1ee73a node DatanodeRegistration(127.0.0.1:45951, datanodeUuid=b611bc95-043b-4517-91a5-22a5c778988f, infoPort=43961, infoSecurePort=0, ipcPort=35285, storageInfo=lv=-57;cid=testClusterID;nsid=342725190;c=1732038476880), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T17:47:59,645 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x690d07acffc94ce3 with lease ID 0xa1c0cea54474c8b: Processing first storage report for DS-b7d45323-c4d5-4c80-aeaa-c091435b72fb from datanode DatanodeRegistration(127.0.0.1:45951, datanodeUuid=b611bc95-043b-4517-91a5-22a5c778988f, infoPort=43961, infoSecurePort=0, ipcPort=35285, storageInfo=lv=-57;cid=testClusterID;nsid=342725190;c=1732038476880) 2024-11-19T17:47:59,645 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x690d07acffc94ce3 with lease ID 0xa1c0cea54474c8b: from storage DS-b7d45323-c4d5-4c80-aeaa-c091435b72fb node DatanodeRegistration(127.0.0.1:45951, datanodeUuid=b611bc95-043b-4517-91a5-22a5c778988f, infoPort=43961, infoSecurePort=0, ipcPort=35285, storageInfo=lv=-57;cid=testClusterID;nsid=342725190;c=1732038476880), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T17:47:59,684 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4 2024-11-19T17:47:59,775 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/zookeeper_0, clientPort=51092, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T17:47:59,787 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51092 2024-11-19T17:47:59,802 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:47:59,805 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:48:00,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741825_1001 (size=7) 2024-11-19T17:48:00,457 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d with version=8 2024-11-19T17:48:00,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/hbase-staging 2024-11-19T17:48:00,544 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T17:48:00,781 INFO [Time-limited test {}] client.ConnectionUtils(128): master/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:48:00,791 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:48:00,791 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:48:00,796 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:48:00,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:48:00,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:48:00,927 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T17:48:00,979 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T17:48:00,990 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T17:48:00,994 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:48:01,019 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21498 (auto-detected) 2024-11-19T17:48:01,019 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T17:48:01,037 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41481 2024-11-19T17:48:01,066 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41481 connecting to ZooKeeper ensemble=127.0.0.1:51092 2024-11-19T17:48:01,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414810x0, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:48:01,210 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41481-0x1015435cb240000 connected 2024-11-19T17:48:01,411 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:48:01,413 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:48:01,422 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:48:01,425 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d, hbase.cluster.distributed=false 2024-11-19T17:48:01,446 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:48:01,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41481 2024-11-19T17:48:01,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41481 2024-11-19T17:48:01,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41481 2024-11-19T17:48:01,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41481 2024-11-19T17:48:01,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41481 2024-11-19T17:48:01,553 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:48:01,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:48:01,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:48:01,555 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:48:01,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:48:01,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:48:01,558 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T17:48:01,561 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:48:01,562 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36839 2024-11-19T17:48:01,563 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36839 connecting to ZooKeeper ensemble=127.0.0.1:51092 2024-11-19T17:48:01,565 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:48:01,568 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:48:01,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368390x0, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:48:01,586 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:368390x0, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:48:01,586 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36839-0x1015435cb240001 connected 2024-11-19T17:48:01,590 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T17:48:01,597 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T17:48:01,599 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T17:48:01,603 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:48:01,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36839 2024-11-19T17:48:01,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36839 2024-11-19T17:48:01,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36839 2024-11-19T17:48:01,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36839 2024-11-19T17:48:01,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36839 2024-11-19T17:48:01,620 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;26d3106abcfe:41481 2024-11-19T17:48:01,621 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/26d3106abcfe,41481,1732038480627 2024-11-19T17:48:01,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:48:01,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:48:01,640 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/26d3106abcfe,41481,1732038480627 2024-11-19T17:48:01,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:01,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T17:48:01,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:01,670 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T17:48:01,671 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/26d3106abcfe,41481,1732038480627 from backup master directory 2024-11-19T17:48:01,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/26d3106abcfe,41481,1732038480627 2024-11-19T17:48:01,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:48:01,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:48:01,683 WARN [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:48:01,684 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=26d3106abcfe,41481,1732038480627 2024-11-19T17:48:01,686 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T17:48:01,687 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T17:48:01,748 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/hbase.id] with ID: 27879909-b964-46e7-ae0b-b7fd34d9080a 2024-11-19T17:48:01,748 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/.tmp/hbase.id 2024-11-19T17:48:01,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741826_1002 (size=42) 2024-11-19T17:48:02,164 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/.tmp/hbase.id]:[hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/hbase.id] 2024-11-19T17:48:02,214 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:48:02,218 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T17:48:02,236 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-19T17:48:02,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:02,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:02,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741827_1003 (size=196) 2024-11-19T17:48:02,727 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:48:02,731 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T17:48:02,752 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:02,758 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:48:02,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741828_1004 (size=1189) 2024-11-19T17:48:03,209 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store 2024-11-19T17:48:03,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741829_1005 (size=34) 2024-11-19T17:48:03,635 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T17:48:03,639 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:03,641 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T17:48:03,641 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:48:03,641 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:48:03,643 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T17:48:03,643 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:48:03,643 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:48:03,644 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732038483640Disabling compacts and flushes for region at 1732038483640Disabling writes for close at 1732038483643 (+3 ms)Writing region close event to WAL at 1732038483643Closed at 1732038483643 2024-11-19T17:48:03,647 WARN [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/.initializing 2024-11-19T17:48:03,647 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/WALs/26d3106abcfe,41481,1732038480627 2024-11-19T17:48:03,656 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:48:03,673 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26d3106abcfe%2C41481%2C1732038480627, suffix=, logDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/WALs/26d3106abcfe,41481,1732038480627, archiveDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/oldWALs, maxLogs=10 2024-11-19T17:48:03,698 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/WALs/26d3106abcfe,41481,1732038480627/26d3106abcfe%2C41481%2C1732038480627.1732038483678, exclude list is [], retry=0 2024-11-19T17:48:03,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45951,DS-e4ab4247-02c2-4707-8ea9-21a06b1ee73a,DISK] 2024-11-19T17:48:03,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T17:48:03,748 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/WALs/26d3106abcfe,41481,1732038480627/26d3106abcfe%2C41481%2C1732038480627.1732038483678 2024-11-19T17:48:03,749 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43961:43961)] 2024-11-19T17:48:03,749 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:48:03,750 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:03,752 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,753 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,788 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T17:48:03,819 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:03,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:03,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T17:48:03,827 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:03,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:03,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T17:48:03,832 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:03,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:03,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T17:48:03,836 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:03,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:03,838 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,841 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,842 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,847 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,848 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,851 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:48:03,856 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:48:03,861 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:48:03,862 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73868525, jitterRate=0.10072679817676544}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:48:03,868 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732038483764Initializing all the Stores at 1732038483766 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038483767 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038483767Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038483768 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038483768Cleaning up temporary data from old regions at 1732038483848 (+80 ms)Region opened successfully at 1732038483868 (+20 ms) 2024-11-19T17:48:03,870 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T17:48:03,905 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27189791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:48:03,931 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T17:48:03,940 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T17:48:03,940 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T17:48:03,942 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T17:48:03,944 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T17:48:03,948 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T17:48:03,948 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T17:48:03,970 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T17:48:03,978 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T17:48:03,998 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T17:48:04,001 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T17:48:04,003 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T17:48:04,009 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T17:48:04,011 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T17:48:04,016 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T17:48:04,027 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T17:48:04,029 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T17:48:04,040 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T17:48:04,062 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T17:48:04,072 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T17:48:04,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:48:04,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:48:04,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:04,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:04,086 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=26d3106abcfe,41481,1732038480627, sessionid=0x1015435cb240000, setting cluster-up flag (Was=false) 2024-11-19T17:48:04,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:04,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:04,146 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T17:48:04,151 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26d3106abcfe,41481,1732038480627 2024-11-19T17:48:04,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:04,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:04,209 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T17:48:04,211 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26d3106abcfe,41481,1732038480627 2024-11-19T17:48:04,219 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T17:48:04,286 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T17:48:04,295 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T17:48:04,300 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T17:48:04,305 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 26d3106abcfe,41481,1732038480627 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T17:48:04,312 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(746): ClusterId : 27879909-b964-46e7-ae0b-b7fd34d9080a 2024-11-19T17:48:04,312 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:48:04,313 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:48:04,313 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:48:04,313 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:48:04,313 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/26d3106abcfe:0, corePoolSize=10, maxPoolSize=10 2024-11-19T17:48:04,313 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,313 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:48:04,314 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,315 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T17:48:04,315 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732038514315 2024-11-19T17:48:04,316 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T17:48:04,317 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T17:48:04,318 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:48:04,319 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T17:48:04,320 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T17:48:04,321 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T17:48:04,321 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T17:48:04,321 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T17:48:04,322 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,324 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:04,324 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T17:48:04,325 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T17:48:04,327 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T17:48:04,327 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T17:48:04,328 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T17:48:04,328 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T17:48:04,329 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T17:48:04,330 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T17:48:04,331 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038484331,5,FailOnTimeoutGroup] 2024-11-19T17:48:04,332 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038484331,5,FailOnTimeoutGroup] 2024-11-19T17:48:04,332 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,332 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T17:48:04,333 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,333 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741831_1007 (size=1321) 2024-11-19T17:48:04,336 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T17:48:04,337 DEBUG [RS:0;26d3106abcfe:36839 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63806d37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:48:04,350 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;26d3106abcfe:36839 2024-11-19T17:48:04,353 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T17:48:04,353 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T17:48:04,353 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T17:48:04,356 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(2659): reportForDuty to master=26d3106abcfe,41481,1732038480627 with port=36839, startcode=1732038481522 2024-11-19T17:48:04,366 DEBUG [RS:0;26d3106abcfe:36839 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T17:48:04,396 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32875, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T17:48:04,402 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41481 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 26d3106abcfe,36839,1732038481522 2024-11-19T17:48:04,404 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41481 {}] master.ServerManager(517): Registering regionserver=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:04,417 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d 2024-11-19T17:48:04,417 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44619 2024-11-19T17:48:04,417 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T17:48:04,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:48:04,431 DEBUG [RS:0;26d3106abcfe:36839 {}] zookeeper.ZKUtil(111): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/26d3106abcfe,36839,1732038481522 2024-11-19T17:48:04,431 WARN [RS:0;26d3106abcfe:36839 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:48:04,431 INFO [RS:0;26d3106abcfe:36839 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:48:04,431 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/WALs/26d3106abcfe,36839,1732038481522 2024-11-19T17:48:04,433 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [26d3106abcfe,36839,1732038481522] 2024-11-19T17:48:04,457 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T17:48:04,472 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T17:48:04,476 INFO [RS:0;26d3106abcfe:36839 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T17:48:04,476 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,477 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T17:48:04,482 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T17:48:04,484 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,484 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,484 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,484 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,484 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,484 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,485 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:48:04,485 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,485 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,485 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,485 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,485 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,486 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:48:04,486 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:48:04,486 DEBUG [RS:0;26d3106abcfe:36839 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:48:04,488 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,488 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,488 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,488 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,488 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,488 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,36839,1732038481522-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:48:04,505 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T17:48:04,507 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,36839,1732038481522-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,507 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,507 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.Replication(171): 26d3106abcfe,36839,1732038481522 started 2024-11-19T17:48:04,524 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:04,524 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1482): Serving as 26d3106abcfe,36839,1732038481522, RpcServer on 26d3106abcfe/172.17.0.2:36839, sessionid=0x1015435cb240001 2024-11-19T17:48:04,525 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T17:48:04,525 DEBUG [RS:0;26d3106abcfe:36839 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 26d3106abcfe,36839,1732038481522 2024-11-19T17:48:04,525 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,36839,1732038481522' 2024-11-19T17:48:04,525 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T17:48:04,526 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T17:48:04,527 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T17:48:04,527 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T17:48:04,527 DEBUG [RS:0;26d3106abcfe:36839 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 26d3106abcfe,36839,1732038481522 2024-11-19T17:48:04,527 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,36839,1732038481522' 2024-11-19T17:48:04,527 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T17:48:04,528 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T17:48:04,528 DEBUG [RS:0;26d3106abcfe:36839 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T17:48:04,528 INFO [RS:0;26d3106abcfe:36839 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T17:48:04,529 INFO [RS:0;26d3106abcfe:36839 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T17:48:04,633 INFO [RS:0;26d3106abcfe:36839 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:48:04,636 INFO [RS:0;26d3106abcfe:36839 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26d3106abcfe%2C36839%2C1732038481522, suffix=, logDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/WALs/26d3106abcfe,36839,1732038481522, archiveDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/oldWALs, maxLogs=32 2024-11-19T17:48:04,652 DEBUG [RS:0;26d3106abcfe:36839 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/WALs/26d3106abcfe,36839,1732038481522/26d3106abcfe%2C36839%2C1732038481522.1732038484639, exclude list is [], retry=0 2024-11-19T17:48:04,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45951,DS-e4ab4247-02c2-4707-8ea9-21a06b1ee73a,DISK] 2024-11-19T17:48:04,660 INFO [RS:0;26d3106abcfe:36839 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/WALs/26d3106abcfe,36839,1732038481522/26d3106abcfe%2C36839%2C1732038481522.1732038484639 2024-11-19T17:48:04,660 DEBUG [RS:0;26d3106abcfe:36839 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43961:43961)] 2024-11-19T17:48:04,739 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T17:48:04,740 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d 2024-11-19T17:48:04,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741833_1009 (size=32) 2024-11-19T17:48:05,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:05,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T17:48:05,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:48:05,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:05,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:05,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T17:48:05,166 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:48:05,166 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:05,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:05,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T17:48:05,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:48:05,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:05,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:05,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T17:48:05,175 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:48:05,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:05,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:05,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:48:05,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740 2024-11-19T17:48:05,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740 2024-11-19T17:48:05,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:48:05,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:48:05,183 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:48:05,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:48:05,189 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:48:05,190 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74709020, jitterRate=0.11325114965438843}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:48:05,192 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732038485155Initializing all the Stores at 1732038485157 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038485157Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038485157Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038485157Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038485157Cleaning up temporary data from old regions at 1732038485182 (+25 ms)Region opened successfully at 1732038485192 (+10 ms) 2024-11-19T17:48:05,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:48:05,193 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:48:05,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:48:05,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:48:05,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:48:05,194 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:48:05,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732038485193Disabling compacts and flushes for region at 1732038485193Disabling writes for close at 1732038485193Writing region close event to WAL at 1732038485194 (+1 ms)Closed at 1732038485194 2024-11-19T17:48:05,197 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:48:05,197 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T17:48:05,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T17:48:05,210 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:48:05,213 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T17:48:05,366 DEBUG [26d3106abcfe:41481 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T17:48:05,378 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:05,385 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26d3106abcfe,36839,1732038481522, state=OPENING 2024-11-19T17:48:05,493 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T17:48:05,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:05,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:48:05,505 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:48:05,505 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:48:05,506 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:48:05,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:48:05,683 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T17:48:05,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37371, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T17:48:05,698 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T17:48:05,699 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:48:05,700 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T17:48:05,704 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26d3106abcfe%2C36839%2C1732038481522.meta, suffix=.meta, logDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/WALs/26d3106abcfe,36839,1732038481522, archiveDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/oldWALs, maxLogs=32 2024-11-19T17:48:05,719 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/WALs/26d3106abcfe,36839,1732038481522/26d3106abcfe%2C36839%2C1732038481522.meta.1732038485706.meta, exclude list is [], retry=0 2024-11-19T17:48:05,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45951,DS-e4ab4247-02c2-4707-8ea9-21a06b1ee73a,DISK] 2024-11-19T17:48:05,726 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/WALs/26d3106abcfe,36839,1732038481522/26d3106abcfe%2C36839%2C1732038481522.meta.1732038485706.meta 2024-11-19T17:48:05,727 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43961:43961)] 2024-11-19T17:48:05,727 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:48:05,729 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T17:48:05,731 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T17:48:05,736 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T17:48:05,740 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T17:48:05,741 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:05,741 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T17:48:05,742 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T17:48:05,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T17:48:05,747 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:48:05,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:05,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:05,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T17:48:05,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:48:05,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:05,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:05,752 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T17:48:05,753 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:48:05,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:05,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:05,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T17:48:05,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:48:05,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:05,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:48:05,758 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:48:05,760 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740 2024-11-19T17:48:05,763 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740 2024-11-19T17:48:05,766 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:48:05,766 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:48:05,767 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:48:05,771 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:48:05,773 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70026001, jitterRate=0.04346872866153717}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:48:05,773 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T17:48:05,775 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732038485742Writing region info on filesystem at 1732038485742Initializing all the Stores at 1732038485744 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038485744Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038485745 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038485745Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038485745Cleaning up temporary data from old regions at 1732038485766 (+21 ms)Running coprocessor post-open hooks at 1732038485773 (+7 ms)Region opened successfully at 1732038485775 (+2 ms) 2024-11-19T17:48:05,782 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732038485675 2024-11-19T17:48:05,795 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T17:48:05,796 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T17:48:05,797 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:05,801 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26d3106abcfe,36839,1732038481522, state=OPEN 2024-11-19T17:48:05,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:48:05,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:48:05,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:48:05,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:48:05,879 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:05,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T17:48:05,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=26d3106abcfe,36839,1732038481522 in 370 msec 2024-11-19T17:48:05,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T17:48:05,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 683 msec 2024-11-19T17:48:05,893 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:48:05,893 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T17:48:05,911 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:05,912 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:05,930 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:05,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50331, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:05,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7000 sec 2024-11-19T17:48:05,950 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732038485950, completionTime=-1 2024-11-19T17:48:05,954 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T17:48:05,954 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T17:48:05,980 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-19T17:48:05,981 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732038545980 2024-11-19T17:48:05,981 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732038605981 2024-11-19T17:48:05,981 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-19T17:48:05,982 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,41481,1732038480627-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:05,983 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,41481,1732038480627-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:05,983 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,41481,1732038480627-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:05,984 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-26d3106abcfe:41481, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:05,984 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:05,985 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:05,992 DEBUG [master/26d3106abcfe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T17:48:06,010 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.326sec 2024-11-19T17:48:06,011 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T17:48:06,012 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T17:48:06,013 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T17:48:06,013 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T17:48:06,013 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T17:48:06,014 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,41481,1732038480627-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:48:06,015 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,41481,1732038480627-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T17:48:06,024 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T17:48:06,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:06,025 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T17:48:06,026 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,41481,1732038480627-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:48:06,026 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T17:48:06,027 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T17:48:06,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:06,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:06,040 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2479a655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:06,052 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:06,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:06,055 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:06,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:06,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:06,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=26d3106abcfe,41481,1732038480627 2024-11-19T17:48:06,097 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=931, ProcessCount=11, AvailableMemoryMB=10482 2024-11-19T17:48:06,124 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:48:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,41481,1732038480627 2024-11-19T17:48:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c1af104 2024-11-19T17:48:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:48:06,138 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:48:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:48:06,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:48:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:06,164 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:48:06,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T17:48:06,167 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:06,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:48:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:48:06,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741835_1011 (size=959) 2024-11-19T17:48:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:48:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:48:06,585 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d 2024-11-19T17:48:06,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741836_1012 (size=53) 2024-11-19T17:48:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:48:06,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:06,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 66eb19cac62d93a8797a177e6c688370, disabling compactions & flushes 2024-11-19T17:48:06,998 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:06,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:06,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. after waiting 0 ms 2024-11-19T17:48:06,999 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:06,999 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:06,999 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66eb19cac62d93a8797a177e6c688370: Waiting for close lock at 1732038486998Disabling compacts and flushes for region at 1732038486998Disabling writes for close at 1732038486998Writing region close event to WAL at 1732038486999 (+1 ms)Closed at 1732038486999 2024-11-19T17:48:07,002 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:48:07,008 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732038487002"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732038487002"}]},"ts":"1732038487002"} 2024-11-19T17:48:07,014 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:48:07,015 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:48:07,018 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038487016"}]},"ts":"1732038487016"} 2024-11-19T17:48:07,022 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:48:07,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, ASSIGN}] 2024-11-19T17:48:07,026 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, ASSIGN 2024-11-19T17:48:07,028 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, ASSIGN; state=OFFLINE, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=false 2024-11-19T17:48:07,180 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=66eb19cac62d93a8797a177e6c688370, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:07,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, ASSIGN because future has completed 2024-11-19T17:48:07,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:48:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:48:07,349 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:07,350 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:48:07,350 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,350 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:07,351 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,351 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,353 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,355 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66eb19cac62d93a8797a177e6c688370 columnFamilyName A 2024-11-19T17:48:07,355 DEBUG [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:07,356 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] regionserver.HStore(327): Store=66eb19cac62d93a8797a177e6c688370/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:07,356 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,358 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66eb19cac62d93a8797a177e6c688370 columnFamilyName B 2024-11-19T17:48:07,359 DEBUG [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:07,359 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] regionserver.HStore(327): Store=66eb19cac62d93a8797a177e6c688370/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:07,360 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,362 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66eb19cac62d93a8797a177e6c688370 columnFamilyName C 2024-11-19T17:48:07,362 DEBUG [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:07,363 INFO [StoreOpener-66eb19cac62d93a8797a177e6c688370-1 {}] regionserver.HStore(327): Store=66eb19cac62d93a8797a177e6c688370/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:07,363 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,364 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,365 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,367 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,368 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,369 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:48:07,371 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,375 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:48:07,376 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 66eb19cac62d93a8797a177e6c688370; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60774427, jitterRate=-0.09439046680927277}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:48:07,377 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:07,378 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 66eb19cac62d93a8797a177e6c688370: Running coprocessor pre-open hook at 1732038487351Writing region info on filesystem at 1732038487351Initializing all the Stores at 1732038487352 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038487352Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038487353 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038487353Cleaning up temporary data from old regions at 1732038487368 (+15 ms)Running coprocessor post-open hooks at 1732038487377 (+9 ms)Region opened successfully at 1732038487377 2024-11-19T17:48:07,380 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., pid=6, masterSystemTime=1732038487338 2024-11-19T17:48:07,383 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:07,384 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:07,385 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=66eb19cac62d93a8797a177e6c688370, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:07,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:48:07,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T17:48:07,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 in 207 msec 2024-11-19T17:48:07,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T17:48:07,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, ASSIGN in 371 msec 2024-11-19T17:48:07,401 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:48:07,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038487401"}]},"ts":"1732038487401"} 2024-11-19T17:48:07,405 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:48:07,407 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:48:07,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2540 sec 2024-11-19T17:48:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:48:08,331 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:08,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,339 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a04f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435bed7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,345 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,351 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:48:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,41481,1732038480627 2024-11-19T17:48:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bf45604 2024-11-19T17:48:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:48:08,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:48:08,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,364 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,366 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,370 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,373 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,377 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,390 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0d5e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,394 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6af04b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,397 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5eb702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,401 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0b2d87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c270ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9550bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,410 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b38edab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,414 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020fb6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,416 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57865cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:08,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:08,422 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463070d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:08,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5c6f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:08,429 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,431 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,437 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,440 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,440 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,441 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,444 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T17:48:08,467 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:08,468 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,469 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:08,471 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,473 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:48:08,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,481 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:08,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:08,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:48:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:08,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:08,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:48:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:48:08,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:08,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/c967ecc3bbf04f44b987044553360e17 is 50, key is test_row_0/A:col10/1732038488552/Put/seqid=0 2024-11-19T17:48:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:08,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:08,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038548682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741837_1013 (size=14341) 2024-11-19T17:48:08,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/c967ecc3bbf04f44b987044553360e17 2024-11-19T17:48:08,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038548692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038548697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038548703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038548708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:48:08,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/d889309c12e94313988f09657ab5d066 is 50, key is test_row_0/B:col10/1732038488552/Put/seqid=0 2024-11-19T17:48:08,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038548862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038548864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038548864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038548864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038548863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:08,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:08,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:08,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:08,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:08,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:08,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741838_1014 (size=12001) 2024-11-19T17:48:08,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/d889309c12e94313988f09657ab5d066 2024-11-19T17:48:08,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/b66ff29003d647efa7fa00cc93f1053a is 50, key is test_row_0/C:col10/1732038488552/Put/seqid=0 2024-11-19T17:48:08,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741839_1015 (size=12001) 2024-11-19T17:48:08,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/b66ff29003d647efa7fa00cc93f1053a 2024-11-19T17:48:08,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/c967ecc3bbf04f44b987044553360e17 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c967ecc3bbf04f44b987044553360e17 2024-11-19T17:48:09,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c967ecc3bbf04f44b987044553360e17, entries=200, sequenceid=14, filesize=14.0 K 2024-11-19T17:48:09,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/d889309c12e94313988f09657ab5d066 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/d889309c12e94313988f09657ab5d066 2024-11-19T17:48:09,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/d889309c12e94313988f09657ab5d066, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T17:48:09,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/b66ff29003d647efa7fa00cc93f1053a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b66ff29003d647efa7fa00cc93f1053a 2024-11-19T17:48:09,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b66ff29003d647efa7fa00cc93f1053a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T17:48:09,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 66eb19cac62d93a8797a177e6c688370 in 509ms, sequenceid=14, compaction requested=false 2024-11-19T17:48:09,066 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:48:09,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:48:09,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:09,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-19T17:48:09,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ed4c2b19cd8a4535ba07bcabf0850b05 is 50, key is test_row_0/A:col10/1732038489105/Put/seqid=0 2024-11-19T17:48:09,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038549121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038549128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038549135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038549140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038549140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741840_1016 (size=14341) 2024-11-19T17:48:09,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ed4c2b19cd8a4535ba07bcabf0850b05 2024-11-19T17:48:09,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:09,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:09,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/77d4b6ac0f104fcc9e2e276e958be231 is 50, key is test_row_0/B:col10/1732038489105/Put/seqid=0 2024-11-19T17:48:09,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038549250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038549250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741841_1017 (size=12001) 2024-11-19T17:48:09,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/77d4b6ac0f104fcc9e2e276e958be231 2024-11-19T17:48:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038549259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038549260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038549261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/a918ca280b1543af8baca0c99fa94b41 is 50, key is test_row_0/C:col10/1732038489105/Put/seqid=0 2024-11-19T17:48:09,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741842_1018 (size=12001) 2024-11-19T17:48:09,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/a918ca280b1543af8baca0c99fa94b41 2024-11-19T17:48:09,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ed4c2b19cd8a4535ba07bcabf0850b05 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ed4c2b19cd8a4535ba07bcabf0850b05 2024-11-19T17:48:09,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ed4c2b19cd8a4535ba07bcabf0850b05, entries=200, sequenceid=43, filesize=14.0 K 2024-11-19T17:48:09,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/77d4b6ac0f104fcc9e2e276e958be231 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/77d4b6ac0f104fcc9e2e276e958be231 2024-11-19T17:48:09,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:09,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:09,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/77d4b6ac0f104fcc9e2e276e958be231, entries=150, sequenceid=43, filesize=11.7 K 2024-11-19T17:48:09,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/a918ca280b1543af8baca0c99fa94b41 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/a918ca280b1543af8baca0c99fa94b41 2024-11-19T17:48:09,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/a918ca280b1543af8baca0c99fa94b41, entries=150, sequenceid=43, filesize=11.7 K 2024-11-19T17:48:09,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 66eb19cac62d93a8797a177e6c688370 in 298ms, sequenceid=43, compaction requested=false 2024-11-19T17:48:09,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:09,483 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:09,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:48:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:09,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/f982baddf25d40a195ae34caee9d567e is 50, key is test_row_0/A:col10/1732038489468/Put/seqid=0 2024-11-19T17:48:09,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:09,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741843_1019 (size=19021) 2024-11-19T17:48:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:48:09,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038549568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038549608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038549610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038549611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038549612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:09,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:09,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038549719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038549729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038549730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038549730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038549731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:09,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:09,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:09,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038549929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/f982baddf25d40a195ae34caee9d567e 2024-11-19T17:48:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038549949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038549949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038549950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038549951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:09,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/63db62db8adc48409a9291c43a30e7a5 is 50, key is test_row_0/B:col10/1732038489468/Put/seqid=0 2024-11-19T17:48:10,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:10,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:10,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741844_1020 (size=12001) 2024-11-19T17:48:10,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/63db62db8adc48409a9291c43a30e7a5 2024-11-19T17:48:10,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/6c40eeb6445948d4a5ac812d223c1d3c is 50, key is test_row_0/C:col10/1732038489468/Put/seqid=0 2024-11-19T17:48:10,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741845_1021 (size=12001) 2024-11-19T17:48:10,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038550239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038550270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038550270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038550270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038550271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:10,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,452 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T17:48:10,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:10,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:10,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/6c40eeb6445948d4a5ac812d223c1d3c 2024-11-19T17:48:10,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/f982baddf25d40a195ae34caee9d567e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/f982baddf25d40a195ae34caee9d567e 2024-11-19T17:48:10,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/f982baddf25d40a195ae34caee9d567e, entries=300, sequenceid=54, filesize=18.6 K 2024-11-19T17:48:10,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/63db62db8adc48409a9291c43a30e7a5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/63db62db8adc48409a9291c43a30e7a5 2024-11-19T17:48:10,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/63db62db8adc48409a9291c43a30e7a5, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T17:48:10,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/6c40eeb6445948d4a5ac812d223c1d3c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6c40eeb6445948d4a5ac812d223c1d3c 2024-11-19T17:48:10,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6c40eeb6445948d4a5ac812d223c1d3c, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T17:48:10,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 66eb19cac62d93a8797a177e6c688370 in 1098ms, sequenceid=54, compaction requested=true 2024-11-19T17:48:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:10,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:10,586 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:10,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:10,589 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:10,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:10,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:10,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:10,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:10,593 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:10,594 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:10,595 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,595 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/d889309c12e94313988f09657ab5d066, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/77d4b6ac0f104fcc9e2e276e958be231, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/63db62db8adc48409a9291c43a30e7a5] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=35.2 K 2024-11-19T17:48:10,597 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d889309c12e94313988f09657ab5d066, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732038488502 2024-11-19T17:48:10,601 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d4b6ac0f104fcc9e2e276e958be231, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732038488702 2024-11-19T17:48:10,601 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:10,601 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:10,601 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,601 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c967ecc3bbf04f44b987044553360e17, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ed4c2b19cd8a4535ba07bcabf0850b05, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/f982baddf25d40a195ae34caee9d567e] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=46.6 K 2024-11-19T17:48:10,601 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 63db62db8adc48409a9291c43a30e7a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732038489468 2024-11-19T17:48:10,602 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c967ecc3bbf04f44b987044553360e17, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732038488502 2024-11-19T17:48:10,604 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4c2b19cd8a4535ba07bcabf0850b05, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732038488702 2024-11-19T17:48:10,610 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting f982baddf25d40a195ae34caee9d567e, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732038489120 2024-11-19T17:48:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:48:10,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:48:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,631 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:48:10,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b9f164224c4f4deda4630bd2d74fc011 is 50, key is test_row_0/A:col10/1732038489562/Put/seqid=0 2024-11-19T17:48:10,679 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#9 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:10,680 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/8a23c4ebe0af422196bfff3b5710cd21 is 50, key is test_row_0/A:col10/1732038489468/Put/seqid=0 2024-11-19T17:48:10,696 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#11 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:10,697 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/45f4d5c2245c458297e72f5dba3b3834 is 50, key is test_row_0/B:col10/1732038489468/Put/seqid=0 2024-11-19T17:48:10,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741846_1022 (size=12104) 2024-11-19T17:48:10,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741847_1023 (size=12001) 2024-11-19T17:48:10,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:10,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741848_1024 (size=12104) 2024-11-19T17:48:10,784 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/8a23c4ebe0af422196bfff3b5710cd21 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8a23c4ebe0af422196bfff3b5710cd21 2024-11-19T17:48:10,800 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/45f4d5c2245c458297e72f5dba3b3834 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/45f4d5c2245c458297e72f5dba3b3834 2024-11-19T17:48:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038550789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038550792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038550798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038550806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038550813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,823 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 8a23c4ebe0af422196bfff3b5710cd21(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:10,824 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:10,828 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into 45f4d5c2245c458297e72f5dba3b3834(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:10,829 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:10,831 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038490588; duration=0sec 2024-11-19T17:48:10,831 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038490583; duration=0sec 2024-11-19T17:48:10,832 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:10,832 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:10,832 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:10,832 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:10,832 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:10,835 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:10,835 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:10,835 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:10,835 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b66ff29003d647efa7fa00cc93f1053a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/a918ca280b1543af8baca0c99fa94b41, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6c40eeb6445948d4a5ac812d223c1d3c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=35.2 K 2024-11-19T17:48:10,837 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b66ff29003d647efa7fa00cc93f1053a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732038488502 2024-11-19T17:48:10,838 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting a918ca280b1543af8baca0c99fa94b41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732038488702 2024-11-19T17:48:10,839 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c40eeb6445948d4a5ac812d223c1d3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732038489468 2024-11-19T17:48:10,870 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:10,871 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/4b3a34a1d39345ddaf0ee24bca2e2f66 is 50, key is test_row_0/C:col10/1732038489468/Put/seqid=0 2024-11-19T17:48:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038550918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038550919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038550929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038550930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:10,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741849_1025 (size=12104) 2024-11-19T17:48:11,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038551139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,147 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b9f164224c4f4deda4630bd2d74fc011 2024-11-19T17:48:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038551140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038551140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038551140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/984b34cc73944f4c8ecde0f8f4b4bb38 is 50, key is test_row_0/B:col10/1732038489562/Put/seqid=0 2024-11-19T17:48:11,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741850_1026 (size=12001) 2024-11-19T17:48:11,255 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:48:11,320 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T17:48:11,321 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T17:48:11,324 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T17:48:11,324 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T17:48:11,325 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T17:48:11,325 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T17:48:11,327 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:48:11,327 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T17:48:11,368 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/4b3a34a1d39345ddaf0ee24bca2e2f66 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/4b3a34a1d39345ddaf0ee24bca2e2f66 2024-11-19T17:48:11,391 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 4b3a34a1d39345ddaf0ee24bca2e2f66(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:11,391 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:11,391 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038490590; duration=0sec 2024-11-19T17:48:11,392 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:11,392 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038551459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038551459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038551460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038551460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,602 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/984b34cc73944f4c8ecde0f8f4b4bb38 2024-11-19T17:48:11,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/cb01b3e453974ed38370dfbb37ebe104 is 50, key is test_row_0/C:col10/1732038489562/Put/seqid=0 2024-11-19T17:48:11,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741851_1027 (size=12001) 2024-11-19T17:48:11,663 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/cb01b3e453974ed38370dfbb37ebe104 2024-11-19T17:48:11,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b9f164224c4f4deda4630bd2d74fc011 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b9f164224c4f4deda4630bd2d74fc011 2024-11-19T17:48:11,694 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b9f164224c4f4deda4630bd2d74fc011, entries=150, sequenceid=79, filesize=11.7 K 2024-11-19T17:48:11,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/984b34cc73944f4c8ecde0f8f4b4bb38 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/984b34cc73944f4c8ecde0f8f4b4bb38 2024-11-19T17:48:11,711 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/984b34cc73944f4c8ecde0f8f4b4bb38, entries=150, sequenceid=79, filesize=11.7 K 2024-11-19T17:48:11,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/cb01b3e453974ed38370dfbb37ebe104 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/cb01b3e453974ed38370dfbb37ebe104 2024-11-19T17:48:11,725 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/cb01b3e453974ed38370dfbb37ebe104, entries=150, sequenceid=79, filesize=11.7 K 2024-11-19T17:48:11,727 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 66eb19cac62d93a8797a177e6c688370 in 1095ms, sequenceid=79, compaction requested=false 2024-11-19T17:48:11,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:11,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:11,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T17:48:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T17:48:11,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T17:48:11,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2600 sec 2024-11-19T17:48:11,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.2800 sec 2024-11-19T17:48:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:11,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:11,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:48:11,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/c0b719f9a541410da732bfee371d3e36 is 50, key is test_row_0/A:col10/1732038491829/Put/seqid=0 2024-11-19T17:48:11,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741852_1028 (size=14341) 2024-11-19T17:48:11,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/c0b719f9a541410da732bfee371d3e36 2024-11-19T17:48:11,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/6468e79fe60747feaefddea97a7e7498 is 50, key is test_row_0/B:col10/1732038491829/Put/seqid=0 2024-11-19T17:48:11,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741853_1029 (size=12001) 2024-11-19T17:48:11,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/6468e79fe60747feaefddea97a7e7498 2024-11-19T17:48:11,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/61562f682d7047a4a076ed80aa41cbba is 50, key is test_row_0/C:col10/1732038491829/Put/seqid=0 2024-11-19T17:48:11,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741854_1030 (size=12001) 2024-11-19T17:48:11,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038551969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038551969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038551974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038551979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038551979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:12,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038552088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038552298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:12,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/61562f682d7047a4a076ed80aa41cbba 2024-11-19T17:48:12,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/c0b719f9a541410da732bfee371d3e36 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c0b719f9a541410da732bfee371d3e36 2024-11-19T17:48:12,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c0b719f9a541410da732bfee371d3e36, entries=200, sequenceid=94, filesize=14.0 K 2024-11-19T17:48:12,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/6468e79fe60747feaefddea97a7e7498 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6468e79fe60747feaefddea97a7e7498 2024-11-19T17:48:12,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6468e79fe60747feaefddea97a7e7498, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:48:12,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/61562f682d7047a4a076ed80aa41cbba as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/61562f682d7047a4a076ed80aa41cbba 2024-11-19T17:48:12,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/61562f682d7047a4a076ed80aa41cbba, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:48:12,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 66eb19cac62d93a8797a177e6c688370 in 611ms, sequenceid=94, compaction requested=true 2024-11-19T17:48:12,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:12,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:12,443 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:12,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:12,443 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:12,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:12,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:12,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:12,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:12,445 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:12,445 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:12,445 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:12,446 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/45f4d5c2245c458297e72f5dba3b3834, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/984b34cc73944f4c8ecde0f8f4b4bb38, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6468e79fe60747feaefddea97a7e7498] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=35.3 K 2024-11-19T17:48:12,446 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:12,446 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:12,446 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:12,446 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8a23c4ebe0af422196bfff3b5710cd21, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b9f164224c4f4deda4630bd2d74fc011, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c0b719f9a541410da732bfee371d3e36] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=37.5 K 2024-11-19T17:48:12,448 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f4d5c2245c458297e72f5dba3b3834, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732038489468 2024-11-19T17:48:12,449 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a23c4ebe0af422196bfff3b5710cd21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732038489468 2024-11-19T17:48:12,449 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 984b34cc73944f4c8ecde0f8f4b4bb38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732038489559 2024-11-19T17:48:12,450 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9f164224c4f4deda4630bd2d74fc011, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732038489559 2024-11-19T17:48:12,450 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 6468e79fe60747feaefddea97a7e7498, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038490799 2024-11-19T17:48:12,450 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0b719f9a541410da732bfee371d3e36, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038490764 2024-11-19T17:48:12,481 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:12,482 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/68fb871e105246e5b7495f549aae8489 is 50, key is test_row_0/B:col10/1732038491829/Put/seqid=0 2024-11-19T17:48:12,485 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:12,489 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/5e8de40c562e410babc8fd83d7d9b421 is 50, key is test_row_0/A:col10/1732038491829/Put/seqid=0 2024-11-19T17:48:12,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741855_1031 (size=12207) 2024-11-19T17:48:12,525 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/68fb871e105246e5b7495f549aae8489 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/68fb871e105246e5b7495f549aae8489 2024-11-19T17:48:12,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741856_1032 (size=12207) 2024-11-19T17:48:12,542 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into 68fb871e105246e5b7495f549aae8489(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:12,542 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/5e8de40c562e410babc8fd83d7d9b421 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5e8de40c562e410babc8fd83d7d9b421 2024-11-19T17:48:12,542 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:12,542 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038492443; duration=0sec 2024-11-19T17:48:12,543 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:12,543 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:12,543 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:12,546 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:12,546 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:12,546 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:12,546 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/4b3a34a1d39345ddaf0ee24bca2e2f66, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/cb01b3e453974ed38370dfbb37ebe104, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/61562f682d7047a4a076ed80aa41cbba] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=35.3 K 2024-11-19T17:48:12,548 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b3a34a1d39345ddaf0ee24bca2e2f66, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732038489468 2024-11-19T17:48:12,551 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting cb01b3e453974ed38370dfbb37ebe104, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732038489559 2024-11-19T17:48:12,554 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 61562f682d7047a4a076ed80aa41cbba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038490799 2024-11-19T17:48:12,558 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 5e8de40c562e410babc8fd83d7d9b421(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:12,559 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:12,559 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038492443; duration=0sec 2024-11-19T17:48:12,559 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:12,559 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:12,587 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:12,588 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9b11176eb42d454a935e347e5755fe54 is 50, key is test_row_0/C:col10/1732038491829/Put/seqid=0 2024-11-19T17:48:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:12,613 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:12,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:48:12,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741857_1033 (size=12207) 2024-11-19T17:48:12,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/41a45b55493f4467965a05215febe91c is 50, key is test_row_0/A:col10/1732038491945/Put/seqid=0 2024-11-19T17:48:12,644 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9b11176eb42d454a935e347e5755fe54 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9b11176eb42d454a935e347e5755fe54 2024-11-19T17:48:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:48:12,649 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:12,650 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T17:48:12,653 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:48:12,655 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:12,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:12,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741858_1034 (size=14341) 2024-11-19T17:48:12,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/41a45b55493f4467965a05215febe91c 2024-11-19T17:48:12,668 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 9b11176eb42d454a935e347e5755fe54(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:12,668 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:12,668 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038492444; duration=0sec 2024-11-19T17:48:12,669 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:12,669 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:12,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4073a60483b343c1b3852d3137ca0b60 is 50, key is test_row_0/B:col10/1732038491945/Put/seqid=0 2024-11-19T17:48:12,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038552687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:12,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741859_1035 (size=12001) 2024-11-19T17:48:12,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4073a60483b343c1b3852d3137ca0b60 2024-11-19T17:48:12,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8d50e67a273241c7b29104bd930ae0b9 is 50, key is test_row_0/C:col10/1732038491945/Put/seqid=0 2024-11-19T17:48:12,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741860_1036 (size=12001) 2024-11-19T17:48:12,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8d50e67a273241c7b29104bd930ae0b9 2024-11-19T17:48:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:48:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/41a45b55493f4467965a05215febe91c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/41a45b55493f4467965a05215febe91c 2024-11-19T17:48:12,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/41a45b55493f4467965a05215febe91c, entries=200, sequenceid=121, filesize=14.0 K 2024-11-19T17:48:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4073a60483b343c1b3852d3137ca0b60 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4073a60483b343c1b3852d3137ca0b60 2024-11-19T17:48:12,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4073a60483b343c1b3852d3137ca0b60, entries=150, sequenceid=121, filesize=11.7 K 2024-11-19T17:48:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8d50e67a273241c7b29104bd930ae0b9 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8d50e67a273241c7b29104bd930ae0b9 2024-11-19T17:48:12,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038552798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:12,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8d50e67a273241c7b29104bd930ae0b9, entries=150, sequenceid=121, filesize=11.7 K 2024-11-19T17:48:12,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 66eb19cac62d93a8797a177e6c688370 in 196ms, sequenceid=121, compaction requested=false 2024-11-19T17:48:12,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:12,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T17:48:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:12,811 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:48:12,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/2a89620bf15a4fa7a73f98a91f9dacb1 is 50, key is test_row_0/A:col10/1732038492685/Put/seqid=0 2024-11-19T17:48:12,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741861_1037 (size=12101) 2024-11-19T17:48:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:48:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:12,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038553044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038553045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038553045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038553048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038553048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038553159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038553159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038553159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038553159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038553159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,241 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/2a89620bf15a4fa7a73f98a91f9dacb1 2024-11-19T17:48:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1ff0e645a59d45c4ae934b9b30498622 is 50, key is test_row_0/B:col10/1732038492685/Put/seqid=0 2024-11-19T17:48:13,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741862_1038 (size=12101) 2024-11-19T17:48:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:48:13,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038553369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038553369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038553369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038553370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038553370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,665 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1ff0e645a59d45c4ae934b9b30498622 2024-11-19T17:48:13,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038553679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038553679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9298641653014f76bbb476c88167efb8 is 50, key is test_row_0/C:col10/1732038492685/Put/seqid=0 2024-11-19T17:48:13,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038553689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038553689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038553689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:13,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741863_1039 (size=12101) 2024-11-19T17:48:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:48:14,107 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9298641653014f76bbb476c88167efb8 2024-11-19T17:48:14,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/2a89620bf15a4fa7a73f98a91f9dacb1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2a89620bf15a4fa7a73f98a91f9dacb1 2024-11-19T17:48:14,126 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2a89620bf15a4fa7a73f98a91f9dacb1, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:48:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1ff0e645a59d45c4ae934b9b30498622 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ff0e645a59d45c4ae934b9b30498622 2024-11-19T17:48:14,136 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ff0e645a59d45c4ae934b9b30498622, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:48:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9298641653014f76bbb476c88167efb8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9298641653014f76bbb476c88167efb8 2024-11-19T17:48:14,151 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9298641653014f76bbb476c88167efb8, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:48:14,153 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 66eb19cac62d93a8797a177e6c688370 in 1342ms, sequenceid=133, compaction requested=true 2024-11-19T17:48:14,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T17:48:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T17:48:14,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T17:48:14,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5010 sec 2024-11-19T17:48:14,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5120 sec 2024-11-19T17:48:14,196 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:14,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T17:48:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:14,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b5d9c79e2fee44f6813242c996cee562 is 50, key is test_row_0/A:col10/1732038493022/Put/seqid=0 2024-11-19T17:48:14,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038554211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038554210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038554213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038554213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038554213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741864_1040 (size=14541) 2024-11-19T17:48:14,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b5d9c79e2fee44f6813242c996cee562 2024-11-19T17:48:14,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/416610d2c2ac4ffab6b4fd1ddf28dc7d is 50, key is test_row_0/B:col10/1732038493022/Put/seqid=0 2024-11-19T17:48:14,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741865_1041 (size=12151) 2024-11-19T17:48:14,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/416610d2c2ac4ffab6b4fd1ddf28dc7d 2024-11-19T17:48:14,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9675dfa8978e4dfe8306e9edfec6708b is 50, key is test_row_0/C:col10/1732038493022/Put/seqid=0 2024-11-19T17:48:14,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741866_1042 (size=12151) 2024-11-19T17:48:14,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038554319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038554319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038554319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038554319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038554320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038554529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038554529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038554529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038554529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038554530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9675dfa8978e4dfe8306e9edfec6708b 2024-11-19T17:48:14,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b5d9c79e2fee44f6813242c996cee562 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5d9c79e2fee44f6813242c996cee562 2024-11-19T17:48:14,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5d9c79e2fee44f6813242c996cee562, entries=200, sequenceid=159, filesize=14.2 K 2024-11-19T17:48:14,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/416610d2c2ac4ffab6b4fd1ddf28dc7d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/416610d2c2ac4ffab6b4fd1ddf28dc7d 2024-11-19T17:48:14,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/416610d2c2ac4ffab6b4fd1ddf28dc7d, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T17:48:14,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9675dfa8978e4dfe8306e9edfec6708b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9675dfa8978e4dfe8306e9edfec6708b 2024-11-19T17:48:14,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9675dfa8978e4dfe8306e9edfec6708b, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T17:48:14,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 66eb19cac62d93a8797a177e6c688370 in 554ms, sequenceid=159, compaction requested=true 2024-11-19T17:48:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:14,751 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:48:14,751 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:14,755 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:14,755 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:14,755 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:14,756 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5e8de40c562e410babc8fd83d7d9b421, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/41a45b55493f4467965a05215febe91c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2a89620bf15a4fa7a73f98a91f9dacb1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5d9c79e2fee44f6813242c996cee562] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=51.9 K 2024-11-19T17:48:14,757 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8de40c562e410babc8fd83d7d9b421, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038490799 2024-11-19T17:48:14,757 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:14,757 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:14,757 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:14,757 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/68fb871e105246e5b7495f549aae8489, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4073a60483b343c1b3852d3137ca0b60, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ff0e645a59d45c4ae934b9b30498622, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/416610d2c2ac4ffab6b4fd1ddf28dc7d] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=47.3 K 2024-11-19T17:48:14,758 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a45b55493f4467965a05215febe91c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732038491945 2024-11-19T17:48:14,758 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 68fb871e105246e5b7495f549aae8489, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038490799 2024-11-19T17:48:14,758 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a89620bf15a4fa7a73f98a91f9dacb1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038492655 2024-11-19T17:48:14,759 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 4073a60483b343c1b3852d3137ca0b60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732038491945 2024-11-19T17:48:14,759 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d9c79e2fee44f6813242c996cee562, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038493022 2024-11-19T17:48:14,760 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ff0e645a59d45c4ae934b9b30498622, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038492655 2024-11-19T17:48:14,761 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 416610d2c2ac4ffab6b4fd1ddf28dc7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038493022 2024-11-19T17:48:14,780 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:14,781 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/4c5346103911491786a44c07af9d26b8 is 50, key is test_row_0/A:col10/1732038493022/Put/seqid=0 2024-11-19T17:48:14,783 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:14,783 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/8d2a3e43461e438890adb9e8af665127 is 50, key is test_row_0/B:col10/1732038493022/Put/seqid=0 2024-11-19T17:48:14,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741867_1043 (size=12493) 2024-11-19T17:48:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:48:14,799 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:14,800 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:14,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741868_1044 (size=12493) 2024-11-19T17:48:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T17:48:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:48:14,803 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:14,805 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:14,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:14,811 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/4c5346103911491786a44c07af9d26b8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/4c5346103911491786a44c07af9d26b8 2024-11-19T17:48:14,831 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 4c5346103911491786a44c07af9d26b8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:14,831 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:14,831 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=12, startTime=1732038494751; duration=0sec 2024-11-19T17:48:14,831 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:14,831 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:14,833 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:14,836 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:14,836 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:14,836 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:14,836 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9b11176eb42d454a935e347e5755fe54, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8d50e67a273241c7b29104bd930ae0b9, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9298641653014f76bbb476c88167efb8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9675dfa8978e4dfe8306e9edfec6708b] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=47.3 K 2024-11-19T17:48:14,837 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b11176eb42d454a935e347e5755fe54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038490799 2024-11-19T17:48:14,838 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d50e67a273241c7b29104bd930ae0b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732038491945 2024-11-19T17:48:14,842 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9298641653014f76bbb476c88167efb8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038492655 2024-11-19T17:48:14,843 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9675dfa8978e4dfe8306e9edfec6708b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038493022 2024-11-19T17:48:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:14,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:14,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:48:14,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/56f4fbcb828d43e69a24d6b938efac3c is 50, key is test_row_0/A:col10/1732038494843/Put/seqid=0 2024-11-19T17:48:14,882 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:14,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/97818f4a77fe4779bbfd3b0f664fae17 is 50, key is test_row_0/C:col10/1732038493022/Put/seqid=0 2024-11-19T17:48:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038554891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038554893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038554897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038554899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:48:14,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038554904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:14,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:14,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741870_1046 (size=12493) 2024-11-19T17:48:14,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741869_1045 (size=12151) 2024-11-19T17:48:14,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/56f4fbcb828d43e69a24d6b938efac3c 2024-11-19T17:48:14,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/bfe1dff327f04e51984bfef2e257f2f4 is 50, key is test_row_0/B:col10/1732038494843/Put/seqid=0 2024-11-19T17:48:14,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T17:48:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:14,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741871_1047 (size=12151) 2024-11-19T17:48:15,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038555008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038555009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038555009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038555009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038555018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T17:48:15,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:15,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:48:15,217 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/8d2a3e43461e438890adb9e8af665127 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/8d2a3e43461e438890adb9e8af665127 2024-11-19T17:48:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038555218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038555229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038555229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,231 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into 8d2a3e43461e438890adb9e8af665127(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,231 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,231 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=12, startTime=1732038494751; duration=0sec 2024-11-19T17:48:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038555229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,231 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:15,232 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:15,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038555231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T17:48:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,351 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/97818f4a77fe4779bbfd3b0f664fae17 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/97818f4a77fe4779bbfd3b0f664fae17 2024-11-19T17:48:15,359 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 97818f4a77fe4779bbfd3b0f664fae17(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:15,359 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:15,359 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=12, startTime=1732038494751; duration=0sec 2024-11-19T17:48:15,359 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:15,360 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:15,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/bfe1dff327f04e51984bfef2e257f2f4 2024-11-19T17:48:15,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T17:48:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:48:15,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/53be3535b742423c92d4f5c12b6dffcb is 50, key is test_row_0/C:col10/1732038494843/Put/seqid=0 2024-11-19T17:48:15,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741872_1048 (size=12151) 2024-11-19T17:48:15,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/53be3535b742423c92d4f5c12b6dffcb 2024-11-19T17:48:15,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/56f4fbcb828d43e69a24d6b938efac3c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/56f4fbcb828d43e69a24d6b938efac3c 2024-11-19T17:48:15,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/56f4fbcb828d43e69a24d6b938efac3c, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T17:48:15,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/bfe1dff327f04e51984bfef2e257f2f4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/bfe1dff327f04e51984bfef2e257f2f4 2024-11-19T17:48:15,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/bfe1dff327f04e51984bfef2e257f2f4, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T17:48:15,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/53be3535b742423c92d4f5c12b6dffcb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/53be3535b742423c92d4f5c12b6dffcb 2024-11-19T17:48:15,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/53be3535b742423c92d4f5c12b6dffcb, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T17:48:15,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 66eb19cac62d93a8797a177e6c688370 in 665ms, sequenceid=172, compaction requested=false 2024-11-19T17:48:15,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:15,545 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:15,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T17:48:15,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038555551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038555554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038555556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038555559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038555560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b5881c021bab430b95cdd9db29077a82 is 50, key is test_row_0/A:col10/1732038495546/Put/seqid=0 2024-11-19T17:48:15,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T17:48:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741873_1049 (size=12151) 2024-11-19T17:48:15,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b5881c021bab430b95cdd9db29077a82 2024-11-19T17:48:15,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/df869f30a7ee415d852a71044b5e512a is 50, key is test_row_0/B:col10/1732038495546/Put/seqid=0 2024-11-19T17:48:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038555670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038555670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038555671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038555671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038555671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:15,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741874_1050 (size=12151) 2024-11-19T17:48:15,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/df869f30a7ee415d852a71044b5e512a 2024-11-19T17:48:15,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/d8fe91ffb48e4925b4f2f77915f81b53 is 50, key is test_row_0/C:col10/1732038495546/Put/seqid=0 2024-11-19T17:48:15,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T17:48:15,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:15,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741875_1051 (size=12151) 2024-11-19T17:48:15,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/d8fe91ffb48e4925b4f2f77915f81b53 2024-11-19T17:48:15,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/b5881c021bab430b95cdd9db29077a82 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5881c021bab430b95cdd9db29077a82 2024-11-19T17:48:15,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5881c021bab430b95cdd9db29077a82, entries=150, sequenceid=203, filesize=11.9 K 2024-11-19T17:48:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/df869f30a7ee415d852a71044b5e512a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/df869f30a7ee415d852a71044b5e512a 2024-11-19T17:48:15,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/df869f30a7ee415d852a71044b5e512a, entries=150, sequenceid=203, filesize=11.9 K 2024-11-19T17:48:15,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/d8fe91ffb48e4925b4f2f77915f81b53 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/d8fe91ffb48e4925b4f2f77915f81b53 2024-11-19T17:48:15,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/d8fe91ffb48e4925b4f2f77915f81b53, entries=150, sequenceid=203, filesize=11.9 K 2024-11-19T17:48:15,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 66eb19cac62d93a8797a177e6c688370 in 275ms, sequenceid=203, compaction requested=true 2024-11-19T17:48:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:15,821 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:15,821 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:15,824 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:15,824 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:15,824 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:15,824 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:15,824 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,824 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,824 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/8d2a3e43461e438890adb9e8af665127, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/bfe1dff327f04e51984bfef2e257f2f4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/df869f30a7ee415d852a71044b5e512a] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=35.9 K 2024-11-19T17:48:15,824 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/4c5346103911491786a44c07af9d26b8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/56f4fbcb828d43e69a24d6b938efac3c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5881c021bab430b95cdd9db29077a82] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=35.9 K 2024-11-19T17:48:15,831 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2a3e43461e438890adb9e8af665127, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038493022 2024-11-19T17:48:15,831 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c5346103911491786a44c07af9d26b8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038493022 2024-11-19T17:48:15,832 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56f4fbcb828d43e69a24d6b938efac3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732038494209 2024-11-19T17:48:15,832 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe1dff327f04e51984bfef2e257f2f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732038494209 2024-11-19T17:48:15,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:15,833 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5881c021bab430b95cdd9db29077a82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732038495542 2024-11-19T17:48:15,833 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting df869f30a7ee415d852a71044b5e512a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732038495542 2024-11-19T17:48:15,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:15,867 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#39 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:15,868 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/e022099490bf403e99b11ed013585116 is 50, key is test_row_0/B:col10/1732038495546/Put/seqid=0 2024-11-19T17:48:15,884 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#40 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:15,885 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0c06c3c93594465a825408263956457a is 50, key is test_row_0/A:col10/1732038495546/Put/seqid=0 2024-11-19T17:48:15,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T17:48:15,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:15,893 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T17:48:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:15,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:48:15,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/335e5953c9d64ebb92fd0532ae390ffc is 50, key is test_row_0/A:col10/1732038495889/Put/seqid=0 2024-11-19T17:48:15,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741876_1052 (size=12595) 2024-11-19T17:48:15,965 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/e022099490bf403e99b11ed013585116 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e022099490bf403e99b11ed013585116 2024-11-19T17:48:15,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741878_1054 (size=16931) 2024-11-19T17:48:15,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741877_1053 (size=12595) 2024-11-19T17:48:15,969 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/335e5953c9d64ebb92fd0532ae390ffc 2024-11-19T17:48:15,987 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0c06c3c93594465a825408263956457a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0c06c3c93594465a825408263956457a 2024-11-19T17:48:15,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/a14fc01607184bf080ce74ea1c73e301 is 50, key is test_row_0/B:col10/1732038495889/Put/seqid=0 2024-11-19T17:48:16,003 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into e022099490bf403e99b11ed013585116(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:16,003 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:16,003 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038495821; duration=0sec 2024-11-19T17:48:16,003 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 0c06c3c93594465a825408263956457a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:16,003 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:16,003 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038495821; duration=0sec 2024-11-19T17:48:16,004 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:16,004 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:16,004 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:16,006 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:16,006 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:16,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741879_1055 (size=12151) 2024-11-19T17:48:16,009 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:16,009 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:16,009 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:16,009 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/97818f4a77fe4779bbfd3b0f664fae17, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/53be3535b742423c92d4f5c12b6dffcb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/d8fe91ffb48e4925b4f2f77915f81b53] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=35.9 K 2024-11-19T17:48:16,010 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97818f4a77fe4779bbfd3b0f664fae17, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038493022 2024-11-19T17:48:16,011 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53be3535b742423c92d4f5c12b6dffcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732038494209 2024-11-19T17:48:16,021 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8fe91ffb48e4925b4f2f77915f81b53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732038495542 2024-11-19T17:48:16,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038556007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038556007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038556032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038556035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038556035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,056 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#43 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:16,057 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/c238ca4355544c3db477f4cca85ed84a is 50, key is test_row_0/C:col10/1732038495546/Put/seqid=0 2024-11-19T17:48:16,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741880_1056 (size=12595) 2024-11-19T17:48:16,098 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/c238ca4355544c3db477f4cca85ed84a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/c238ca4355544c3db477f4cca85ed84a 2024-11-19T17:48:16,108 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into c238ca4355544c3db477f4cca85ed84a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:16,109 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:16,109 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038495834; duration=0sec 2024-11-19T17:48:16,109 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:16,109 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:16,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038556140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038556139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038556149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038556149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038556149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038556349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038556358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038556359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038556359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038556360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,410 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/a14fc01607184bf080ce74ea1c73e301 2024-11-19T17:48:16,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/905bc21c70454136972b822a4763b584 is 50, key is test_row_0/C:col10/1732038495889/Put/seqid=0 2024-11-19T17:48:16,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741881_1057 (size=12151) 2024-11-19T17:48:16,455 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/905bc21c70454136972b822a4763b584 2024-11-19T17:48:16,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/335e5953c9d64ebb92fd0532ae390ffc as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/335e5953c9d64ebb92fd0532ae390ffc 2024-11-19T17:48:16,479 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/335e5953c9d64ebb92fd0532ae390ffc, entries=250, sequenceid=214, filesize=16.5 K 2024-11-19T17:48:16,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/a14fc01607184bf080ce74ea1c73e301 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a14fc01607184bf080ce74ea1c73e301 2024-11-19T17:48:16,516 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a14fc01607184bf080ce74ea1c73e301, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T17:48:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/905bc21c70454136972b822a4763b584 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/905bc21c70454136972b822a4763b584 2024-11-19T17:48:16,524 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/905bc21c70454136972b822a4763b584, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T17:48:16,527 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 66eb19cac62d93a8797a177e6c688370 in 634ms, sequenceid=214, compaction requested=false 2024-11-19T17:48:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T17:48:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T17:48:16,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T17:48:16,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7250 sec 2024-11-19T17:48:16,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7350 sec 2024-11-19T17:48:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:16,671 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:16,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T17:48:16,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/247a4bff44084704af7bbed1b7f06705 is 50, key is test_row_0/A:col10/1732038496670/Put/seqid=0 2024-11-19T17:48:16,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038556682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038556685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038556690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038556692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038556692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741882_1058 (size=16931) 2024-11-19T17:48:16,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/247a4bff44084704af7bbed1b7f06705 2024-11-19T17:48:16,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1a0f04e6f97140d28368d172f5fdfeaa is 50, key is test_row_0/B:col10/1732038496670/Put/seqid=0 2024-11-19T17:48:16,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741883_1059 (size=12151) 2024-11-19T17:48:16,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038556799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038556799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038556809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038556809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038556809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:48:16,949 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:16,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T17:48:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:48:16,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:16,954 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:16,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038557010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038557010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038557019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038557020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038557020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:48:17,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:48:17,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:17,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1a0f04e6f97140d28368d172f5fdfeaa 2024-11-19T17:48:17,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/3d01ad6643944d8999df0fe9e7e9ede6 is 50, key is test_row_0/C:col10/1732038496670/Put/seqid=0 2024-11-19T17:48:17,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741884_1060 (size=12151) 2024-11-19T17:48:17,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:48:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:48:17,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038557319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038557319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038557328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038557329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038557338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:48:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:48:17,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:48:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:17,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/3d01ad6643944d8999df0fe9e7e9ede6 2024-11-19T17:48:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/247a4bff44084704af7bbed1b7f06705 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/247a4bff44084704af7bbed1b7f06705 2024-11-19T17:48:17,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/247a4bff44084704af7bbed1b7f06705, entries=250, sequenceid=246, filesize=16.5 K 2024-11-19T17:48:17,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1a0f04e6f97140d28368d172f5fdfeaa as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1a0f04e6f97140d28368d172f5fdfeaa 2024-11-19T17:48:17,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1a0f04e6f97140d28368d172f5fdfeaa, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T17:48:17,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/3d01ad6643944d8999df0fe9e7e9ede6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d01ad6643944d8999df0fe9e7e9ede6 2024-11-19T17:48:17,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d01ad6643944d8999df0fe9e7e9ede6, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T17:48:17,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 66eb19cac62d93a8797a177e6c688370 in 992ms, sequenceid=246, compaction requested=true 2024-11-19T17:48:17,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:17,663 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:17,665 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 46457 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:17,665 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:17,665 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,665 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0c06c3c93594465a825408263956457a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/335e5953c9d64ebb92fd0532ae390ffc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/247a4bff44084704af7bbed1b7f06705] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=45.4 K 2024-11-19T17:48:17,666 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c06c3c93594465a825408263956457a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732038495542 2024-11-19T17:48:17,666 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 335e5953c9d64ebb92fd0532ae390ffc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038495555 2024-11-19T17:48:17,667 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 247a4bff44084704af7bbed1b7f06705, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732038496031 2024-11-19T17:48:17,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:17,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:17,673 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:17,675 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:17,675 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:17,675 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,676 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e022099490bf403e99b11ed013585116, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a14fc01607184bf080ce74ea1c73e301, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1a0f04e6f97140d28368d172f5fdfeaa] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=36.0 K 2024-11-19T17:48:17,676 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting e022099490bf403e99b11ed013585116, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732038495542 2024-11-19T17:48:17,677 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a14fc01607184bf080ce74ea1c73e301, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038495556 2024-11-19T17:48:17,677 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a0f04e6f97140d28368d172f5fdfeaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732038496033 2024-11-19T17:48:17,693 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:17,694 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/c006a3827ccc4b6880feb6ba68a30270 is 50, key is test_row_0/A:col10/1732038496670/Put/seqid=0 2024-11-19T17:48:17,705 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:17,705 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/db04c30d931b44c3913e9cd17209661b is 50, key is test_row_0/B:col10/1732038496670/Put/seqid=0 2024-11-19T17:48:17,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741885_1061 (size=12697) 2024-11-19T17:48:17,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741886_1062 (size=12697) 2024-11-19T17:48:17,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:48:17,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:17,739 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T17:48:17,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0854f9edf5b74beba74eafbd8c1f60af is 50, key is test_row_0/A:col10/1732038496687/Put/seqid=0 2024-11-19T17:48:17,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741887_1063 (size=12151) 2024-11-19T17:48:17,756 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0854f9edf5b74beba74eafbd8c1f60af 2024-11-19T17:48:17,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/6c5c978b9ee14ae783a5d3d82c2c54ea is 50, key is test_row_0/B:col10/1732038496687/Put/seqid=0 2024-11-19T17:48:17,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741888_1064 (size=12151) 2024-11-19T17:48:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038557882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038557887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038557895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038557897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:17,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038557897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038557998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038558009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038558009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038558009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038558009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:48:18,127 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/c006a3827ccc4b6880feb6ba68a30270 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c006a3827ccc4b6880feb6ba68a30270 2024-11-19T17:48:18,135 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into c006a3827ccc4b6880feb6ba68a30270(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:18,135 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:18,135 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038497663; duration=0sec 2024-11-19T17:48:18,135 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:18,135 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:18,135 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:18,140 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:18,141 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:18,141 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:18,141 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/c238ca4355544c3db477f4cca85ed84a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/905bc21c70454136972b822a4763b584, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d01ad6643944d8999df0fe9e7e9ede6] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=36.0 K 2024-11-19T17:48:18,142 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c238ca4355544c3db477f4cca85ed84a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732038495542 2024-11-19T17:48:18,144 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 905bc21c70454136972b822a4763b584, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038495556 2024-11-19T17:48:18,144 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d01ad6643944d8999df0fe9e7e9ede6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732038496033 2024-11-19T17:48:18,148 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/db04c30d931b44c3913e9cd17209661b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/db04c30d931b44c3913e9cd17209661b 2024-11-19T17:48:18,156 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into db04c30d931b44c3913e9cd17209661b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:18,157 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:18,157 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038497673; duration=0sec 2024-11-19T17:48:18,157 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:18,157 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:18,158 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:18,159 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/7257f8789f5e4162bd1b209ca5aa95bf is 50, key is test_row_0/C:col10/1732038496670/Put/seqid=0 2024-11-19T17:48:18,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741889_1065 (size=12697) 2024-11-19T17:48:18,195 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/6c5c978b9ee14ae783a5d3d82c2c54ea 2024-11-19T17:48:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/54947c680a5b4d63b1e3eb2b979eaa04 is 50, key is test_row_0/C:col10/1732038496687/Put/seqid=0 2024-11-19T17:48:18,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038558208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038558218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038558219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038558219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741890_1066 (size=12151) 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038558219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038558518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038558542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038558542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038558542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038558543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:18,576 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/7257f8789f5e4162bd1b209ca5aa95bf as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/7257f8789f5e4162bd1b209ca5aa95bf 2024-11-19T17:48:18,583 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 7257f8789f5e4162bd1b209ca5aa95bf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:18,583 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:18,583 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038497673; duration=0sec 2024-11-19T17:48:18,584 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:18,584 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:18,624 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/54947c680a5b4d63b1e3eb2b979eaa04 2024-11-19T17:48:18,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0854f9edf5b74beba74eafbd8c1f60af as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0854f9edf5b74beba74eafbd8c1f60af 2024-11-19T17:48:18,636 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0854f9edf5b74beba74eafbd8c1f60af, entries=150, sequenceid=254, filesize=11.9 K 2024-11-19T17:48:18,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/6c5c978b9ee14ae783a5d3d82c2c54ea as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6c5c978b9ee14ae783a5d3d82c2c54ea 2024-11-19T17:48:18,643 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6c5c978b9ee14ae783a5d3d82c2c54ea, entries=150, sequenceid=254, filesize=11.9 K 2024-11-19T17:48:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/54947c680a5b4d63b1e3eb2b979eaa04 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/54947c680a5b4d63b1e3eb2b979eaa04 2024-11-19T17:48:18,651 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/54947c680a5b4d63b1e3eb2b979eaa04, entries=150, sequenceid=254, filesize=11.9 K 2024-11-19T17:48:18,652 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=181.14 KB/185490 for 66eb19cac62d93a8797a177e6c688370 in 914ms, sequenceid=254, compaction requested=false 2024-11-19T17:48:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T17:48:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T17:48:18,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T17:48:18,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7010 sec 2024-11-19T17:48:18,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7090 sec 2024-11-19T17:48:19,030 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:19,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-19T17:48:19,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ffc34ec4cabf41ef8f25ad806aa605ba is 50, key is test_row_0/A:col10/1732038497879/Put/seqid=0 2024-11-19T17:48:19,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038559039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:19,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038559048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:19,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038559048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038559048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:19,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741891_1067 (size=14741) 2024-11-19T17:48:19,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ffc34ec4cabf41ef8f25ad806aa605ba 2024-11-19T17:48:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038559059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:19,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4ca7bd79ae674f98b003df0465863e80 is 50, key is test_row_0/B:col10/1732038497879/Put/seqid=0 2024-11-19T17:48:19,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741892_1068 (size=12301) 2024-11-19T17:48:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:48:19,099 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:19,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T17:48:19,101 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:48:19,102 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:19,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038559149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:48:19,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:48:19,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:19,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038559358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:19,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:48:19,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:19,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:48:19,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4ca7bd79ae674f98b003df0465863e80 2024-11-19T17:48:19,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9fcc7b30a8f24e9d9e74851651e39289 is 50, key is test_row_0/C:col10/1732038497879/Put/seqid=0 2024-11-19T17:48:19,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741893_1069 (size=12301) 2024-11-19T17:48:19,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:48:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038559668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:19,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:48:19,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:48:19,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:48:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:19,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9fcc7b30a8f24e9d9e74851651e39289 2024-11-19T17:48:19,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ffc34ec4cabf41ef8f25ad806aa605ba as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ffc34ec4cabf41ef8f25ad806aa605ba 2024-11-19T17:48:19,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ffc34ec4cabf41ef8f25ad806aa605ba, entries=200, sequenceid=288, filesize=14.4 K 2024-11-19T17:48:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4ca7bd79ae674f98b003df0465863e80 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4ca7bd79ae674f98b003df0465863e80 2024-11-19T17:48:19,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4ca7bd79ae674f98b003df0465863e80, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T17:48:19,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/9fcc7b30a8f24e9d9e74851651e39289 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9fcc7b30a8f24e9d9e74851651e39289 2024-11-19T17:48:19,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9fcc7b30a8f24e9d9e74851651e39289, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T17:48:19,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 66eb19cac62d93a8797a177e6c688370 in 924ms, sequenceid=288, compaction requested=true 2024-11-19T17:48:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:19,955 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:19,955 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:19,957 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:19,957 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:19,957 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:19,957 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:19,957 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,957 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:19,957 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/db04c30d931b44c3913e9cd17209661b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6c5c978b9ee14ae783a5d3d82c2c54ea, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4ca7bd79ae674f98b003df0465863e80] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=36.3 K 2024-11-19T17:48:19,957 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c006a3827ccc4b6880feb6ba68a30270, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0854f9edf5b74beba74eafbd8c1f60af, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ffc34ec4cabf41ef8f25ad806aa605ba] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=38.7 K 2024-11-19T17:48:19,958 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c006a3827ccc4b6880feb6ba68a30270, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732038496033 2024-11-19T17:48:19,958 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting db04c30d931b44c3913e9cd17209661b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732038496033 2024-11-19T17:48:19,958 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0854f9edf5b74beba74eafbd8c1f60af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732038496683 2024-11-19T17:48:19,958 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5c978b9ee14ae783a5d3d82c2c54ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732038496683 2024-11-19T17:48:19,959 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc34ec4cabf41ef8f25ad806aa605ba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038497879 2024-11-19T17:48:19,959 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca7bd79ae674f98b003df0465863e80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038497879 2024-11-19T17:48:19,979 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:19,980 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/a1f080b5b0cf4021b52fb909386cb075 is 50, key is test_row_0/B:col10/1732038497879/Put/seqid=0 2024-11-19T17:48:19,999 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:19,999 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/1915428845454b53add056a9f231d994 is 50, key is test_row_0/A:col10/1732038497879/Put/seqid=0 2024-11-19T17:48:20,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741894_1070 (size=12949) 2024-11-19T17:48:20,026 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/a1f080b5b0cf4021b52fb909386cb075 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a1f080b5b0cf4021b52fb909386cb075 2024-11-19T17:48:20,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:48:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,027 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-19T17:48:20,039 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into a1f080b5b0cf4021b52fb909386cb075(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:20,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741895_1071 (size=12949) 2024-11-19T17:48:20,039 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:20,039 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038499955; duration=0sec 2024-11-19T17:48:20,039 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:20,039 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:20,039 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:20,041 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:20,041 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:20,042 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,042 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/7257f8789f5e4162bd1b209ca5aa95bf, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/54947c680a5b4d63b1e3eb2b979eaa04, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9fcc7b30a8f24e9d9e74851651e39289] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=36.3 K 2024-11-19T17:48:20,044 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 7257f8789f5e4162bd1b209ca5aa95bf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732038496033 2024-11-19T17:48:20,045 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 54947c680a5b4d63b1e3eb2b979eaa04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732038496683 2024-11-19T17:48:20,048 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fcc7b30a8f24e9d9e74851651e39289, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038497879 2024-11-19T17:48:20,053 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/1915428845454b53add056a9f231d994 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1915428845454b53add056a9f231d994 2024-11-19T17:48:20,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/abe883833494447287c2bf743243fd77 is 50, key is test_row_0/A:col10/1732038499033/Put/seqid=0 2024-11-19T17:48:20,067 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 1915428845454b53add056a9f231d994(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:20,067 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:20,067 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038499955; duration=0sec 2024-11-19T17:48:20,067 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:20,067 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:20,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741896_1072 (size=9857) 2024-11-19T17:48:20,081 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/abe883833494447287c2bf743243fd77 2024-11-19T17:48:20,081 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:20,082 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/b457c8cfd539456e901909bb134fe845 is 50, key is test_row_0/C:col10/1732038497879/Put/seqid=0 2024-11-19T17:48:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:20,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/e48b97a47ed04d7ca5480194ecb8eb36 is 50, key is test_row_0/B:col10/1732038499033/Put/seqid=0 2024-11-19T17:48:20,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741897_1073 (size=12949) 2024-11-19T17:48:20,116 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/b457c8cfd539456e901909bb134fe845 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b457c8cfd539456e901909bb134fe845 2024-11-19T17:48:20,126 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into b457c8cfd539456e901909bb134fe845(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:20,126 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:20,126 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038499955; duration=0sec 2024-11-19T17:48:20,126 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:20,126 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:20,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741898_1074 (size=9857) 2024-11-19T17:48:20,133 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/e48b97a47ed04d7ca5480194ecb8eb36 2024-11-19T17:48:20,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038560141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038560142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038560143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038560145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/59b7b36ea3674c49a7c856a6154220c2 is 50, key is test_row_0/C:col10/1732038499033/Put/seqid=0 2024-11-19T17:48:20,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741899_1075 (size=9857) 2024-11-19T17:48:20,176 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/59b7b36ea3674c49a7c856a6154220c2 2024-11-19T17:48:20,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038560179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/abe883833494447287c2bf743243fd77 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/abe883833494447287c2bf743243fd77 2024-11-19T17:48:20,196 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/abe883833494447287c2bf743243fd77, entries=100, sequenceid=293, filesize=9.6 K 2024-11-19T17:48:20,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/e48b97a47ed04d7ca5480194ecb8eb36 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e48b97a47ed04d7ca5480194ecb8eb36 2024-11-19T17:48:20,207 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e48b97a47ed04d7ca5480194ecb8eb36, entries=100, sequenceid=293, filesize=9.6 K 2024-11-19T17:48:20,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/59b7b36ea3674c49a7c856a6154220c2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/59b7b36ea3674c49a7c856a6154220c2 2024-11-19T17:48:20,221 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/59b7b36ea3674c49a7c856a6154220c2, entries=100, sequenceid=293, filesize=9.6 K 2024-11-19T17:48:20,222 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=194.56 KB/199230 for 66eb19cac62d93a8797a177e6c688370 in 195ms, sequenceid=293, compaction requested=false 2024-11-19T17:48:20,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:20,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T17:48:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T17:48:20,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T17:48:20,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1280 sec 2024-11-19T17:48:20,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.1350 sec 2024-11-19T17:48:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:48:20,250 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:20,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T17:48:20,254 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:48:20,255 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:20,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:20,257 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:20,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-19T17:48:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:20,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038560259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038560261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038560262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038560263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ccbb8e49ce20457caa99a0877f00a21c is 50, key is test_row_0/A:col10/1732038500141/Put/seqid=0 2024-11-19T17:48:20,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741900_1076 (size=12301) 2024-11-19T17:48:20,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ccbb8e49ce20457caa99a0877f00a21c 2024-11-19T17:48:20,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4474865e306745fe92640ff8e4df25f0 is 50, key is test_row_0/B:col10/1732038500141/Put/seqid=0 2024-11-19T17:48:20,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741901_1077 (size=12301) 2024-11-19T17:48:20,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4474865e306745fe92640ff8e4df25f0 2024-11-19T17:48:20,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/bfe1993830874b3e9f54c44622f1b64e is 50, key is test_row_0/C:col10/1732038500141/Put/seqid=0 2024-11-19T17:48:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:48:20,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038560369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741902_1078 (size=12301) 2024-11-19T17:48:20,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T17:48:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:20,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038560468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038560468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038560479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T17:48:20,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:20,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:48:20,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038560579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T17:48:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:20,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:20,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/bfe1993830874b3e9f54c44622f1b64e 2024-11-19T17:48:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038560779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038560779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038560789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:20,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/ccbb8e49ce20457caa99a0877f00a21c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ccbb8e49ce20457caa99a0877f00a21c 2024-11-19T17:48:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:20,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ccbb8e49ce20457caa99a0877f00a21c, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T17:48:20,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/4474865e306745fe92640ff8e4df25f0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4474865e306745fe92640ff8e4df25f0 2024-11-19T17:48:20,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4474865e306745fe92640ff8e4df25f0, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T17:48:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/bfe1993830874b3e9f54c44622f1b64e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/bfe1993830874b3e9f54c44622f1b64e 2024-11-19T17:48:20,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/bfe1993830874b3e9f54c44622f1b64e, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T17:48:20,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 66eb19cac62d93a8797a177e6c688370 in 576ms, sequenceid=329, compaction requested=true 2024-11-19T17:48:20,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:20,834 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:20,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:20,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:20,834 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:20,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:20,836 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:20,836 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:20,836 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,836 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1915428845454b53add056a9f231d994, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/abe883833494447287c2bf743243fd77, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ccbb8e49ce20457caa99a0877f00a21c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=34.3 K 2024-11-19T17:48:20,836 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:20,837 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:20,837 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,837 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a1f080b5b0cf4021b52fb909386cb075, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e48b97a47ed04d7ca5480194ecb8eb36, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4474865e306745fe92640ff8e4df25f0] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=34.3 K 2024-11-19T17:48:20,837 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1915428845454b53add056a9f231d994, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038497879 2024-11-19T17:48:20,837 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f080b5b0cf4021b52fb909386cb075, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038497879 2024-11-19T17:48:20,838 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting abe883833494447287c2bf743243fd77, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732038499033 2024-11-19T17:48:20,838 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting e48b97a47ed04d7ca5480194ecb8eb36, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732038499033 2024-11-19T17:48:20,838 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 4474865e306745fe92640ff8e4df25f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732038500129 2024-11-19T17:48:20,838 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccbb8e49ce20457caa99a0877f00a21c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732038500129 2024-11-19T17:48:20,850 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:20,850 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/afb88348c9004347b75a47e41eb7b518 is 50, key is test_row_0/A:col10/1732038500141/Put/seqid=0 2024-11-19T17:48:20,858 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:20,859 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b4caf318826e4f869c338e7f7b834b60 is 50, key is test_row_0/B:col10/1732038500141/Put/seqid=0 2024-11-19T17:48:20,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T17:48:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T17:48:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T17:48:20,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T17:48:20,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 620 msec 2024-11-19T17:48:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:48:20,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741903_1079 (size=13051) 2024-11-19T17:48:20,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741904_1080 (size=13051) 2024-11-19T17:48:20,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 628 msec 2024-11-19T17:48:20,960 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/afb88348c9004347b75a47e41eb7b518 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/afb88348c9004347b75a47e41eb7b518 2024-11-19T17:48:20,960 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b4caf318826e4f869c338e7f7b834b60 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b4caf318826e4f869c338e7f7b834b60 2024-11-19T17:48:20,979 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into afb88348c9004347b75a47e41eb7b518(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:20,979 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:20,979 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038500834; duration=0sec 2024-11-19T17:48:20,980 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:20,980 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:20,980 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:20,981 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:20,981 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into b4caf318826e4f869c338e7f7b834b60(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:20,981 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:20,981 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:20,981 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038500834; duration=0sec 2024-11-19T17:48:20,981 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:20,981 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:20,981 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b457c8cfd539456e901909bb134fe845, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/59b7b36ea3674c49a7c856a6154220c2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/bfe1993830874b3e9f54c44622f1b64e] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=34.3 K 2024-11-19T17:48:20,981 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:20,982 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b457c8cfd539456e901909bb134fe845, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038497879 2024-11-19T17:48:20,982 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59b7b36ea3674c49a7c856a6154220c2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732038499033 2024-11-19T17:48:20,983 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfe1993830874b3e9f54c44622f1b64e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732038500129 2024-11-19T17:48:21,002 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:21,002 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/86f8ab936a47416e8fc8629a3f24829c is 50, key is test_row_0/C:col10/1732038500141/Put/seqid=0 2024-11-19T17:48:21,006 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:21,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:48:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:21,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/62c1d766b8fa454280665a388f6d8b78 is 50, key is test_row_0/A:col10/1732038500997/Put/seqid=0 2024-11-19T17:48:21,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741905_1081 (size=13051) 2024-11-19T17:48:21,059 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/86f8ab936a47416e8fc8629a3f24829c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/86f8ab936a47416e8fc8629a3f24829c 2024-11-19T17:48:21,072 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 86f8ab936a47416e8fc8629a3f24829c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:21,072 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:21,072 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038500835; duration=0sec 2024-11-19T17:48:21,073 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:21,073 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:21,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741906_1082 (size=17181) 2024-11-19T17:48:21,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/62c1d766b8fa454280665a388f6d8b78 2024-11-19T17:48:21,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1eec565377bd451bbfaa0bad8f82c2f8 is 50, key is test_row_0/B:col10/1732038500997/Put/seqid=0 2024-11-19T17:48:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741907_1083 (size=12301) 2024-11-19T17:48:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038561140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038561188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,193 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 66eb19cac62d93a8797a177e6c688370 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:21,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038561248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:21,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038561289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038561289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038561298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:48:21,389 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:21,390 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T17:48:21,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:21,394 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:21,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:48:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038561459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:48:21,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1eec565377bd451bbfaa0bad8f82c2f8 2024-11-19T17:48:21,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8ef6304de1d2427a9173f049f015f50d is 50, key is test_row_0/C:col10/1732038500997/Put/seqid=0 2024-11-19T17:48:21,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741908_1084 (size=12301) 2024-11-19T17:48:21,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T17:48:21,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:21,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8ef6304de1d2427a9173f049f015f50d 2024-11-19T17:48:21,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:21,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:21,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:21,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:21,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/62c1d766b8fa454280665a388f6d8b78 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/62c1d766b8fa454280665a388f6d8b78 2024-11-19T17:48:21,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/62c1d766b8fa454280665a388f6d8b78, entries=250, sequenceid=343, filesize=16.8 K 2024-11-19T17:48:21,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1eec565377bd451bbfaa0bad8f82c2f8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1eec565377bd451bbfaa0bad8f82c2f8 2024-11-19T17:48:21,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1eec565377bd451bbfaa0bad8f82c2f8, entries=150, sequenceid=343, filesize=12.0 K 2024-11-19T17:48:21,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8ef6304de1d2427a9173f049f015f50d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8ef6304de1d2427a9173f049f015f50d 2024-11-19T17:48:21,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8ef6304de1d2427a9173f049f015f50d, entries=150, sequenceid=343, filesize=12.0 K 2024-11-19T17:48:21,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 66eb19cac62d93a8797a177e6c688370 in 592ms, sequenceid=343, compaction requested=false 2024-11-19T17:48:21,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:21,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T17:48:21,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:21,702 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:48:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:48:21,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0b1b5c8c2163469abb381d4d9d1d8366 is 50, key is test_row_0/A:col10/1732038501126/Put/seqid=0 2024-11-19T17:48:21,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741909_1085 (size=12301) 2024-11-19T17:48:21,725 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0b1b5c8c2163469abb381d4d9d1d8366 2024-11-19T17:48:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/818851b1dceb4a0abee38f1714f4a476 is 50, key is test_row_0/B:col10/1732038501126/Put/seqid=0 2024-11-19T17:48:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741910_1086 (size=12301) 2024-11-19T17:48:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038561795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:21,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038561909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:48:22,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038562119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,149 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/818851b1dceb4a0abee38f1714f4a476 2024-11-19T17:48:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/599ffaf0f5c74951a3396b7a7cd5ad3d is 50, key is test_row_0/C:col10/1732038501126/Put/seqid=0 2024-11-19T17:48:22,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741911_1087 (size=12301) 2024-11-19T17:48:22,164 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/599ffaf0f5c74951a3396b7a7cd5ad3d 2024-11-19T17:48:22,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0b1b5c8c2163469abb381d4d9d1d8366 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0b1b5c8c2163469abb381d4d9d1d8366 2024-11-19T17:48:22,176 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0b1b5c8c2163469abb381d4d9d1d8366, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T17:48:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/818851b1dceb4a0abee38f1714f4a476 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/818851b1dceb4a0abee38f1714f4a476 2024-11-19T17:48:22,183 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/818851b1dceb4a0abee38f1714f4a476, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T17:48:22,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/599ffaf0f5c74951a3396b7a7cd5ad3d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/599ffaf0f5c74951a3396b7a7cd5ad3d 2024-11-19T17:48:22,191 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/599ffaf0f5c74951a3396b7a7cd5ad3d, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T17:48:22,192 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 66eb19cac62d93a8797a177e6c688370 in 489ms, sequenceid=369, compaction requested=true 2024-11-19T17:48:22,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:22,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:22,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T17:48:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T17:48:22,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T17:48:22,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 798 msec 2024-11-19T17:48:22,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 806 msec 2024-11-19T17:48:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:22,301 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:22,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:48:22,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/620aa1b2b6944a629981cb7cb50c8af8 is 50, key is test_row_0/A:col10/1732038501790/Put/seqid=0 2024-11-19T17:48:22,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741912_1088 (size=14741) 2024-11-19T17:48:22,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/620aa1b2b6944a629981cb7cb50c8af8 2024-11-19T17:48:22,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1ca557c693714c22b224eb7545bc2a95 is 50, key is test_row_0/B:col10/1732038501790/Put/seqid=0 2024-11-19T17:48:22,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038562349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038562351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038562355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741913_1089 (size=12301) 2024-11-19T17:48:22,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1ca557c693714c22b224eb7545bc2a95 2024-11-19T17:48:22,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8595c0c2723c4f76b88f2d3f2198736f is 50, key is test_row_0/C:col10/1732038501790/Put/seqid=0 2024-11-19T17:48:22,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741914_1090 (size=12301) 2024-11-19T17:48:22,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038562429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038562458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038562458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038562469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:48:22,528 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:22,529 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T17:48:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:48:22,531 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:22,532 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:22,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:48:22,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038562669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038562669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038562678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T17:48:22,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:22,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:22,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:22,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:22,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:22,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8595c0c2723c4f76b88f2d3f2198736f 2024-11-19T17:48:22,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/620aa1b2b6944a629981cb7cb50c8af8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/620aa1b2b6944a629981cb7cb50c8af8 2024-11-19T17:48:22,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/620aa1b2b6944a629981cb7cb50c8af8, entries=200, sequenceid=381, filesize=14.4 K 2024-11-19T17:48:22,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/1ca557c693714c22b224eb7545bc2a95 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ca557c693714c22b224eb7545bc2a95 2024-11-19T17:48:22,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ca557c693714c22b224eb7545bc2a95, entries=150, sequenceid=381, filesize=12.0 K 2024-11-19T17:48:22,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/8595c0c2723c4f76b88f2d3f2198736f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8595c0c2723c4f76b88f2d3f2198736f 2024-11-19T17:48:22,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8595c0c2723c4f76b88f2d3f2198736f, entries=150, sequenceid=381, filesize=12.0 K 2024-11-19T17:48:22,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 66eb19cac62d93a8797a177e6c688370 in 530ms, sequenceid=381, compaction requested=true 2024-11-19T17:48:22,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:22,832 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:22,833 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:22,834 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:22,834 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:22,834 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:22,834 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/afb88348c9004347b75a47e41eb7b518, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/62c1d766b8fa454280665a388f6d8b78, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0b1b5c8c2163469abb381d4d9d1d8366, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/620aa1b2b6944a629981cb7cb50c8af8] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=55.9 K 2024-11-19T17:48:22,835 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb88348c9004347b75a47e41eb7b518, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732038500129 2024-11-19T17:48:22,835 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:22,835 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:22,835 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:22,835 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b4caf318826e4f869c338e7f7b834b60, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1eec565377bd451bbfaa0bad8f82c2f8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/818851b1dceb4a0abee38f1714f4a476, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ca557c693714c22b224eb7545bc2a95] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=48.8 K 2024-11-19T17:48:22,836 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c1d766b8fa454280665a388f6d8b78, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732038500890 2024-11-19T17:48:22,836 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b4caf318826e4f869c338e7f7b834b60, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732038500129 2024-11-19T17:48:22,837 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eec565377bd451bbfaa0bad8f82c2f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732038500972 2024-11-19T17:48:22,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T17:48:22,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:22,838 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:48:22,838 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b1b5c8c2163469abb381d4d9d1d8366, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732038501116 2024-11-19T17:48:22,838 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 818851b1dceb4a0abee38f1714f4a476, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732038501116 2024-11-19T17:48:22,841 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 620aa1b2b6944a629981cb7cb50c8af8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732038501781 2024-11-19T17:48:22,841 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ca557c693714c22b224eb7545bc2a95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732038501790 2024-11-19T17:48:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:48:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/58cb72e3d051490683b7c185f28b1267 is 50, key is test_row_0/A:col10/1732038502347/Put/seqid=0 2024-11-19T17:48:22,856 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:22,856 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/496881be4d4542fa8a78d65f220a3705 is 50, key is test_row_0/B:col10/1732038501790/Put/seqid=0 2024-11-19T17:48:22,868 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#80 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:22,869 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/63ba20eac9c44e88bee436c7a3c0f3ec is 50, key is test_row_0/A:col10/1732038501790/Put/seqid=0 2024-11-19T17:48:22,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741915_1091 (size=12301) 2024-11-19T17:48:22,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741916_1092 (size=13187) 2024-11-19T17:48:22,891 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/496881be4d4542fa8a78d65f220a3705 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/496881be4d4542fa8a78d65f220a3705 2024-11-19T17:48:22,901 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into 496881be4d4542fa8a78d65f220a3705(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:22,901 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:22,901 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=12, startTime=1732038502833; duration=0sec 2024-11-19T17:48:22,902 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:22,902 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:22,902 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:22,904 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:22,904 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:22,904 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:22,904 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/86f8ab936a47416e8fc8629a3f24829c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8ef6304de1d2427a9173f049f015f50d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/599ffaf0f5c74951a3396b7a7cd5ad3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8595c0c2723c4f76b88f2d3f2198736f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=48.8 K 2024-11-19T17:48:22,905 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f8ab936a47416e8fc8629a3f24829c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732038500129 2024-11-19T17:48:22,905 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ef6304de1d2427a9173f049f015f50d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732038500972 2024-11-19T17:48:22,906 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 599ffaf0f5c74951a3396b7a7cd5ad3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732038501116 2024-11-19T17:48:22,906 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8595c0c2723c4f76b88f2d3f2198736f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732038501790 2024-11-19T17:48:22,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741917_1093 (size=13187) 2024-11-19T17:48:22,917 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/63ba20eac9c44e88bee436c7a3c0f3ec as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/63ba20eac9c44e88bee436c7a3c0f3ec 2024-11-19T17:48:22,926 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 63ba20eac9c44e88bee436c7a3c0f3ec(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:22,926 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:22,926 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=12, startTime=1732038502832; duration=0sec 2024-11-19T17:48:22,926 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:22,927 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:22,939 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:22,939 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/6883b51446a3405dadb7c1e846552309 is 50, key is test_row_0/C:col10/1732038501790/Put/seqid=0 2024-11-19T17:48:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:22,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:22,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038562978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038562978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038562981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:22,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741918_1094 (size=13187) 2024-11-19T17:48:22,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038562989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:23,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038563099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:48:23,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038563209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:23,214 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 66eb19cac62d93a8797a177e6c688370 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:23,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:23,274 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/58cb72e3d051490683b7c185f28b1267 2024-11-19T17:48:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/c662b9ee4f6a40b48f36d9d14b3a9a36 is 50, key is test_row_0/B:col10/1732038502347/Put/seqid=0 2024-11-19T17:48:23,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741919_1095 (size=12301) 2024-11-19T17:48:23,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038563309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:23,396 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/6883b51446a3405dadb7c1e846552309 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6883b51446a3405dadb7c1e846552309 2024-11-19T17:48:23,404 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 6883b51446a3405dadb7c1e846552309(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:23,404 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:23,404 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=12, startTime=1732038502833; duration=0sec 2024-11-19T17:48:23,404 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:23,404 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038563488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038563489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:23,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038563499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:23,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038563619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:48:23,698 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/c662b9ee4f6a40b48f36d9d14b3a9a36 2024-11-19T17:48:23,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/2e4626d35b5f401ab1d845aeb1cda3de is 50, key is test_row_0/C:col10/1732038502347/Put/seqid=0 2024-11-19T17:48:23,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741920_1096 (size=12301) 2024-11-19T17:48:24,114 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/2e4626d35b5f401ab1d845aeb1cda3de 2024-11-19T17:48:24,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/58cb72e3d051490683b7c185f28b1267 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/58cb72e3d051490683b7c185f28b1267 2024-11-19T17:48:24,129 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/58cb72e3d051490683b7c185f28b1267, entries=150, sequenceid=405, filesize=12.0 K 2024-11-19T17:48:24,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038564129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/c662b9ee4f6a40b48f36d9d14b3a9a36 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/c662b9ee4f6a40b48f36d9d14b3a9a36 2024-11-19T17:48:24,143 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/c662b9ee4f6a40b48f36d9d14b3a9a36, entries=150, sequenceid=405, filesize=12.0 K 2024-11-19T17:48:24,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/2e4626d35b5f401ab1d845aeb1cda3de as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/2e4626d35b5f401ab1d845aeb1cda3de 2024-11-19T17:48:24,153 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/2e4626d35b5f401ab1d845aeb1cda3de, entries=150, sequenceid=405, filesize=12.0 K 2024-11-19T17:48:24,155 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 66eb19cac62d93a8797a177e6c688370 in 1317ms, sequenceid=405, compaction requested=false 2024-11-19T17:48:24,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:24,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:24,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T17:48:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T17:48:24,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T17:48:24,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6250 sec 2024-11-19T17:48:24,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6310 sec 2024-11-19T17:48:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:24,503 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:24,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:48:24,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/1e380271febc40a2b15779c74c3f4bf5 is 50, key is test_row_0/A:col10/1732038504501/Put/seqid=0 2024-11-19T17:48:24,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741921_1097 (size=12301) 2024-11-19T17:48:24,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/1e380271febc40a2b15779c74c3f4bf5 2024-11-19T17:48:24,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/20e30f6acf4145af8ff113efc18a189b is 50, key is test_row_0/B:col10/1732038504501/Put/seqid=0 2024-11-19T17:48:24,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038564563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038564567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038564569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741922_1098 (size=12301) 2024-11-19T17:48:24,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/20e30f6acf4145af8ff113efc18a189b 2024-11-19T17:48:24,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/916dfc40675744b890964bc002cc7826 is 50, key is test_row_0/C:col10/1732038504501/Put/seqid=0 2024-11-19T17:48:24,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741923_1099 (size=12301) 2024-11-19T17:48:24,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/916dfc40675744b890964bc002cc7826 2024-11-19T17:48:24,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/1e380271febc40a2b15779c74c3f4bf5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1e380271febc40a2b15779c74c3f4bf5 2024-11-19T17:48:24,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1e380271febc40a2b15779c74c3f4bf5, entries=150, sequenceid=421, filesize=12.0 K 2024-11-19T17:48:24,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/20e30f6acf4145af8ff113efc18a189b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/20e30f6acf4145af8ff113efc18a189b 2024-11-19T17:48:24,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/20e30f6acf4145af8ff113efc18a189b, entries=150, sequenceid=421, filesize=12.0 K 2024-11-19T17:48:24,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/916dfc40675744b890964bc002cc7826 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/916dfc40675744b890964bc002cc7826 2024-11-19T17:48:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038564679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/916dfc40675744b890964bc002cc7826, entries=150, sequenceid=421, filesize=12.0 K 2024-11-19T17:48:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:48:24,685 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:24,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:24,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 66eb19cac62d93a8797a177e6c688370 in 182ms, sequenceid=421, compaction requested=true 2024-11-19T17:48:24,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:24,686 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:24,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:24,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:24,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:24,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:24,687 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:24,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:24,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T17:48:24,688 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:24,688 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:24,688 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:24,688 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:24,688 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:24,688 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:24,688 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/63ba20eac9c44e88bee436c7a3c0f3ec, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/58cb72e3d051490683b7c185f28b1267, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1e380271febc40a2b15779c74c3f4bf5] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=36.9 K 2024-11-19T17:48:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:24,689 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/496881be4d4542fa8a78d65f220a3705, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/c662b9ee4f6a40b48f36d9d14b3a9a36, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/20e30f6acf4145af8ff113efc18a189b] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=36.9 K 2024-11-19T17:48:24,689 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:24,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:48:24,689 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:24,689 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 496881be4d4542fa8a78d65f220a3705, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732038501790 2024-11-19T17:48:24,689 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63ba20eac9c44e88bee436c7a3c0f3ec, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732038501790 2024-11-19T17:48:24,690 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c662b9ee4f6a40b48f36d9d14b3a9a36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732038502347 2024-11-19T17:48:24,690 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:24,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:24,690 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58cb72e3d051490683b7c185f28b1267, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732038502347 2024-11-19T17:48:24,690 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e30f6acf4145af8ff113efc18a189b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732038502961 2024-11-19T17:48:24,691 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e380271febc40a2b15779c74c3f4bf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732038502961 2024-11-19T17:48:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:48:24,710 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:24,711 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/0e93ff8ba2814469b55785f7f53a24ce is 50, key is test_row_0/B:col10/1732038504501/Put/seqid=0 2024-11-19T17:48:24,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038564715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038564717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,722 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:24,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/352ef66eb5924883b55eb020e40a8ea5 is 50, key is test_row_0/A:col10/1732038504561/Put/seqid=0 2024-11-19T17:48:24,723 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/9e00b5843ef04ab3853c4e99908c4e16 is 50, key is test_row_0/A:col10/1732038504501/Put/seqid=0 2024-11-19T17:48:24,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741926_1102 (size=13289) 2024-11-19T17:48:24,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741924_1100 (size=13289) 2024-11-19T17:48:24,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741925_1101 (size=14741) 2024-11-19T17:48:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:48:24,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038564828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038564828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:24,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:24,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038564889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:24,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:24,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:48:25,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038565038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038565038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,150 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/9e00b5843ef04ab3853c4e99908c4e16 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/9e00b5843ef04ab3853c4e99908c4e16 2024-11-19T17:48:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038565149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:25,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,152 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 66eb19cac62d93a8797a177e6c688370 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,161 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 9e00b5843ef04ab3853c4e99908c4e16(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:25,161 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:25,161 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038504686; duration=0sec 2024-11-19T17:48:25,162 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:25,162 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:25,162 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:25,163 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:25,163 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:25,163 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,163 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6883b51446a3405dadb7c1e846552309, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/2e4626d35b5f401ab1d845aeb1cda3de, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/916dfc40675744b890964bc002cc7826] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=36.9 K 2024-11-19T17:48:25,163 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6883b51446a3405dadb7c1e846552309, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732038501790 2024-11-19T17:48:25,164 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e4626d35b5f401ab1d845aeb1cda3de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732038502347 2024-11-19T17:48:25,164 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 916dfc40675744b890964bc002cc7826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732038502961 2024-11-19T17:48:25,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/352ef66eb5924883b55eb020e40a8ea5 2024-11-19T17:48:25,175 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/0e93ff8ba2814469b55785f7f53a24ce as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0e93ff8ba2814469b55785f7f53a24ce 2024-11-19T17:48:25,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/cf3dd564e5ab4a5690915a0fc91192a7 is 50, key is test_row_0/B:col10/1732038504561/Put/seqid=0 2024-11-19T17:48:25,188 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#91 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:25,189 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/5243f919f2094514b13f09d63a83025d is 50, key is test_row_0/C:col10/1732038504501/Put/seqid=0 2024-11-19T17:48:25,190 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into 0e93ff8ba2814469b55785f7f53a24ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:25,190 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:25,190 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038504687; duration=0sec 2024-11-19T17:48:25,190 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:25,190 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:25,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038565198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741928_1104 (size=13289) 2024-11-19T17:48:25,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741927_1103 (size=12301) 2024-11-19T17:48:25,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:25,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:25,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:48:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038565349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038565349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,607 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/5243f919f2094514b13f09d63a83025d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/5243f919f2094514b13f09d63a83025d 2024-11-19T17:48:25,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/cf3dd564e5ab4a5690915a0fc91192a7 2024-11-19T17:48:25,613 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 5243f919f2094514b13f09d63a83025d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:25,614 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:25,614 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038504687; duration=0sec 2024-11-19T17:48:25,614 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:25,614 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:25,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/3d061808f10d4ea0821948fe90e054dd is 50, key is test_row_0/C:col10/1732038504561/Put/seqid=0 2024-11-19T17:48:25,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741929_1105 (size=12301) 2024-11-19T17:48:25,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/3d061808f10d4ea0821948fe90e054dd 2024-11-19T17:48:25,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/352ef66eb5924883b55eb020e40a8ea5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/352ef66eb5924883b55eb020e40a8ea5 2024-11-19T17:48:25,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/352ef66eb5924883b55eb020e40a8ea5, entries=200, sequenceid=446, filesize=14.4 K 2024-11-19T17:48:25,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/cf3dd564e5ab4a5690915a0fc91192a7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/cf3dd564e5ab4a5690915a0fc91192a7 2024-11-19T17:48:25,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/cf3dd564e5ab4a5690915a0fc91192a7, entries=150, sequenceid=446, filesize=12.0 K 2024-11-19T17:48:25,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/3d061808f10d4ea0821948fe90e054dd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d061808f10d4ea0821948fe90e054dd 2024-11-19T17:48:25,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d061808f10d4ea0821948fe90e054dd, entries=150, sequenceid=446, filesize=12.0 K 2024-11-19T17:48:25,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 66eb19cac62d93a8797a177e6c688370 in 962ms, sequenceid=446, compaction requested=false 2024-11-19T17:48:25,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:25,711 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:25,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:48:25,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0f5d42af0dc34786b38220b73e2d2d22 is 50, key is test_row_0/A:col10/1732038505710/Put/seqid=0 2024-11-19T17:48:25,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741930_1106 (size=14741) 2024-11-19T17:48:25,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038565779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:48:25,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038565859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038565859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038565888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:25,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:25,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:26,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:26,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:26,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:26,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:26,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:26,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038566099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:26,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0f5d42af0dc34786b38220b73e2d2d22 2024-11-19T17:48:26,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/0c04391185ef426e94762285ebcb1d15 is 50, key is test_row_0/B:col10/1732038505710/Put/seqid=0 2024-11-19T17:48:26,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741931_1107 (size=12301) 2024-11-19T17:48:26,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/0c04391185ef426e94762285ebcb1d15 2024-11-19T17:48:26,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/618a3b21ba4d40abba5e46343832ac64 is 50, key is test_row_0/C:col10/1732038505710/Put/seqid=0 2024-11-19T17:48:26,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741932_1108 (size=12301) 2024-11-19T17:48:26,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/618a3b21ba4d40abba5e46343832ac64 2024-11-19T17:48:26,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/0f5d42af0dc34786b38220b73e2d2d22 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0f5d42af0dc34786b38220b73e2d2d22 2024-11-19T17:48:26,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0f5d42af0dc34786b38220b73e2d2d22, entries=200, sequenceid=461, filesize=14.4 K 2024-11-19T17:48:26,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/0c04391185ef426e94762285ebcb1d15 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0c04391185ef426e94762285ebcb1d15 2024-11-19T17:48:26,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0c04391185ef426e94762285ebcb1d15, entries=150, sequenceid=461, filesize=12.0 K 2024-11-19T17:48:26,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/618a3b21ba4d40abba5e46343832ac64 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/618a3b21ba4d40abba5e46343832ac64 2024-11-19T17:48:26,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/618a3b21ba4d40abba5e46343832ac64, entries=150, sequenceid=461, filesize=12.0 K 2024-11-19T17:48:26,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 66eb19cac62d93a8797a177e6c688370 in 517ms, sequenceid=461, compaction requested=true 2024-11-19T17:48:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:26,228 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:26,228 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:26,230 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:26,230 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:26,230 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:26,230 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:26,230 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:26,230 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:26,230 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/9e00b5843ef04ab3853c4e99908c4e16, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/352ef66eb5924883b55eb020e40a8ea5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0f5d42af0dc34786b38220b73e2d2d22] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=41.8 K 2024-11-19T17:48:26,230 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0e93ff8ba2814469b55785f7f53a24ce, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/cf3dd564e5ab4a5690915a0fc91192a7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0c04391185ef426e94762285ebcb1d15] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=37.0 K 2024-11-19T17:48:26,230 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e93ff8ba2814469b55785f7f53a24ce, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732038502961 2024-11-19T17:48:26,231 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3dd564e5ab4a5690915a0fc91192a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732038504543 2024-11-19T17:48:26,231 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c04391185ef426e94762285ebcb1d15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732038504713 2024-11-19T17:48:26,238 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e00b5843ef04ab3853c4e99908c4e16, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732038502961 2024-11-19T17:48:26,248 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 352ef66eb5924883b55eb020e40a8ea5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732038504543 2024-11-19T17:48:26,249 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5d42af0dc34786b38220b73e2d2d22, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732038504704 2024-11-19T17:48:26,252 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:26,252 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/3edfed4c5dda4079858c211f175b40e3 is 50, key is test_row_0/B:col10/1732038505710/Put/seqid=0 2024-11-19T17:48:26,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:48:26,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:26,254 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:48:26,265 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#97 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:26,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/e2507a948cf248a3b40eb137da1d73e7 is 50, key is test_row_0/A:col10/1732038505773/Put/seqid=0 2024-11-19T17:48:26,265 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/fa7cd9ecb9f94241b1bcf89d1fd5ce35 is 50, key is test_row_0/A:col10/1732038505710/Put/seqid=0 2024-11-19T17:48:26,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741933_1109 (size=13391) 2024-11-19T17:48:26,276 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/3edfed4c5dda4079858c211f175b40e3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/3edfed4c5dda4079858c211f175b40e3 2024-11-19T17:48:26,285 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into 3edfed4c5dda4079858c211f175b40e3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:26,285 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:26,285 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038506228; duration=0sec 2024-11-19T17:48:26,285 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:26,285 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:26,285 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:26,287 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:26,287 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:26,287 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:26,287 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/5243f919f2094514b13f09d63a83025d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d061808f10d4ea0821948fe90e054dd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/618a3b21ba4d40abba5e46343832ac64] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=37.0 K 2024-11-19T17:48:26,288 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 5243f919f2094514b13f09d63a83025d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732038502961 2024-11-19T17:48:26,289 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d061808f10d4ea0821948fe90e054dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732038504543 2024-11-19T17:48:26,289 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 618a3b21ba4d40abba5e46343832ac64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732038504713 2024-11-19T17:48:26,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741934_1110 (size=12301) 2024-11-19T17:48:26,306 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:26,308 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/77282042f3774589bb359f183ce464ee is 50, key is test_row_0/C:col10/1732038505710/Put/seqid=0 2024-11-19T17:48:26,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741935_1111 (size=13391) 2024-11-19T17:48:26,324 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/fa7cd9ecb9f94241b1bcf89d1fd5ce35 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/fa7cd9ecb9f94241b1bcf89d1fd5ce35 2024-11-19T17:48:26,339 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into fa7cd9ecb9f94241b1bcf89d1fd5ce35(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:26,339 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:26,339 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038506228; duration=0sec 2024-11-19T17:48:26,339 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:26,339 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:26,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741936_1112 (size=13391) 2024-11-19T17:48:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:26,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038566443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038566559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:26,698 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/e2507a948cf248a3b40eb137da1d73e7 2024-11-19T17:48:26,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b19b30054ea84bf7ac5fbbf8faeaae6f is 50, key is test_row_0/B:col10/1732038505773/Put/seqid=0 2024-11-19T17:48:26,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741937_1113 (size=12301) 2024-11-19T17:48:26,724 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b19b30054ea84bf7ac5fbbf8faeaae6f 2024-11-19T17:48:26,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/918625448004493898923f0866911970 is 50, key is test_row_0/C:col10/1732038505773/Put/seqid=0 2024-11-19T17:48:26,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741938_1114 (size=12301) 2024-11-19T17:48:26,756 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/918625448004493898923f0866911970 2024-11-19T17:48:26,761 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/77282042f3774589bb359f183ce464ee as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/77282042f3774589bb359f183ce464ee 2024-11-19T17:48:26,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/e2507a948cf248a3b40eb137da1d73e7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e2507a948cf248a3b40eb137da1d73e7 2024-11-19T17:48:26,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038566768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:26,772 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e2507a948cf248a3b40eb137da1d73e7, entries=150, sequenceid=485, filesize=12.0 K 2024-11-19T17:48:26,773 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 77282042f3774589bb359f183ce464ee(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:26,773 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:26,773 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038506228; duration=0sec 2024-11-19T17:48:26,773 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:26,773 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b19b30054ea84bf7ac5fbbf8faeaae6f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b19b30054ea84bf7ac5fbbf8faeaae6f 2024-11-19T17:48:26,783 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b19b30054ea84bf7ac5fbbf8faeaae6f, entries=150, sequenceid=485, filesize=12.0 K 2024-11-19T17:48:26,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/918625448004493898923f0866911970 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/918625448004493898923f0866911970 2024-11-19T17:48:26,808 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/918625448004493898923f0866911970, entries=150, sequenceid=485, filesize=12.0 K 2024-11-19T17:48:26,817 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 66eb19cac62d93a8797a177e6c688370 in 563ms, sequenceid=485, compaction requested=false 2024-11-19T17:48:26,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:26,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:26,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T17:48:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T17:48:26,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T17:48:26,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1290 sec 2024-11-19T17:48:26,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1360 sec 2024-11-19T17:48:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:48:26,839 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:26,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-19T17:48:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:48:26,842 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:26,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:26,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:26,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T17:48:26,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/8fb911675eab4891b998b3cf7f38b499 is 50, key is test_row_0/A:col10/1732038506414/Put/seqid=0 2024-11-19T17:48:26,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741939_1115 (size=12301) 2024-11-19T17:48:26,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/8fb911675eab4891b998b3cf7f38b499 2024-11-19T17:48:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038566923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038566920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:26,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/f2c80dfd0ba0461aadbdde9974d868fc is 50, key is test_row_0/B:col10/1732038506414/Put/seqid=0 2024-11-19T17:48:26,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741940_1116 (size=12301) 2024-11-19T17:48:26,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/f2c80dfd0ba0461aadbdde9974d868fc 2024-11-19T17:48:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:48:26,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/834937c620024d2da419865a818a2545 is 50, key is test_row_0/C:col10/1732038506414/Put/seqid=0 2024-11-19T17:48:26,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741941_1117 (size=12301) 2024-11-19T17:48:27,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:48:27,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:27,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038567039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038567039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038567078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:48:27,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51506 deadline: 1732038567168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,171 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 66eb19cac62d93a8797a177e6c688370 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:48:27,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:27,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1732038567219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,223 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 66eb19cac62d93a8797a177e6c688370 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:27,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038567248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038567248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:48:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:27,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/834937c620024d2da419865a818a2545 2024-11-19T17:48:27,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/8fb911675eab4891b998b3cf7f38b499 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8fb911675eab4891b998b3cf7f38b499 2024-11-19T17:48:27,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8fb911675eab4891b998b3cf7f38b499, entries=150, sequenceid=502, filesize=12.0 K 2024-11-19T17:48:27,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/f2c80dfd0ba0461aadbdde9974d868fc as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/f2c80dfd0ba0461aadbdde9974d868fc 2024-11-19T17:48:27,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/f2c80dfd0ba0461aadbdde9974d868fc, entries=150, sequenceid=502, filesize=12.0 K 2024-11-19T17:48:27,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/834937c620024d2da419865a818a2545 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/834937c620024d2da419865a818a2545 2024-11-19T17:48:27,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/834937c620024d2da419865a818a2545, entries=150, sequenceid=502, filesize=12.0 K 2024-11-19T17:48:27,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 66eb19cac62d93a8797a177e6c688370 in 530ms, sequenceid=502, compaction requested=true 2024-11-19T17:48:27,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:27,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:27,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:27,401 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:27,401 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:27,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:27,403 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:27,403 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:27,403 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:27,403 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:27,403 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,403 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/3edfed4c5dda4079858c211f175b40e3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b19b30054ea84bf7ac5fbbf8faeaae6f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/f2c80dfd0ba0461aadbdde9974d868fc] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=37.1 K 2024-11-19T17:48:27,403 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,403 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/fa7cd9ecb9f94241b1bcf89d1fd5ce35, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e2507a948cf248a3b40eb137da1d73e7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8fb911675eab4891b998b3cf7f38b499] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=37.1 K 2024-11-19T17:48:27,404 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3edfed4c5dda4079858c211f175b40e3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732038504713 2024-11-19T17:48:27,404 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7cd9ecb9f94241b1bcf89d1fd5ce35, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732038504713 2024-11-19T17:48:27,404 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b19b30054ea84bf7ac5fbbf8faeaae6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732038505748 2024-11-19T17:48:27,404 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2507a948cf248a3b40eb137da1d73e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732038505748 2024-11-19T17:48:27,404 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c80dfd0ba0461aadbdde9974d868fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732038506414 2024-11-19T17:48:27,405 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fb911675eab4891b998b3cf7f38b499, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732038506414 2024-11-19T17:48:27,413 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:27,414 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/114deef2eaf04eff8dc40b022d8d024f is 50, key is test_row_0/A:col10/1732038506414/Put/seqid=0 2024-11-19T17:48:27,426 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:27,427 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b0eef43b10004969a3d9acba54472b69 is 50, key is test_row_0/B:col10/1732038506414/Put/seqid=0 2024-11-19T17:48:27,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741942_1118 (size=13493) 2024-11-19T17:48:27,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741943_1119 (size=13493) 2024-11-19T17:48:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:48:27,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:48:27,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,497 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T17:48:27,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/133f6459804a406f8c1a2d351a1bbae5 is 50, key is test_row_0/A:col10/1732038506902/Put/seqid=0 2024-11-19T17:48:27,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741944_1120 (size=12301) 2024-11-19T17:48:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:27,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. as already flushing 2024-11-19T17:48:27,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038567579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038567580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038567588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038567688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038567689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,840 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/114deef2eaf04eff8dc40b022d8d024f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/114deef2eaf04eff8dc40b022d8d024f 2024-11-19T17:48:27,849 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 114deef2eaf04eff8dc40b022d8d024f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:27,849 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:27,849 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038507401; duration=0sec 2024-11-19T17:48:27,850 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:27,850 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:27,850 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:27,853 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:27,853 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b0eef43b10004969a3d9acba54472b69 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b0eef43b10004969a3d9acba54472b69 2024-11-19T17:48:27,853 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:27,853 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:27,853 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/77282042f3774589bb359f183ce464ee, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/918625448004493898923f0866911970, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/834937c620024d2da419865a818a2545] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=37.1 K 2024-11-19T17:48:27,854 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77282042f3774589bb359f183ce464ee, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732038504713 2024-11-19T17:48:27,854 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918625448004493898923f0866911970, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732038505748 2024-11-19T17:48:27,856 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834937c620024d2da419865a818a2545, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732038506414 2024-11-19T17:48:27,861 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into b0eef43b10004969a3d9acba54472b69(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:27,861 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:27,861 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038507401; duration=0sec 2024-11-19T17:48:27,861 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:27,862 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:27,869 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:27,869 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/abfa619687844c5b9dd6d082a8960af2 is 50, key is test_row_0/C:col10/1732038506414/Put/seqid=0 2024-11-19T17:48:27,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741945_1121 (size=13493) 2024-11-19T17:48:27,899 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/abfa619687844c5b9dd6d082a8960af2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/abfa619687844c5b9dd6d082a8960af2 2024-11-19T17:48:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038567899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038567899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:27,907 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/133f6459804a406f8c1a2d351a1bbae5 2024-11-19T17:48:27,917 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into abfa619687844c5b9dd6d082a8960af2(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:27,917 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:27,917 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038507402; duration=0sec 2024-11-19T17:48:27,918 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:27,918 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:27,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b5e70f2bbd154bc29a4810e4d5bbe694 is 50, key is test_row_0/B:col10/1732038506902/Put/seqid=0 2024-11-19T17:48:27,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741946_1122 (size=12301) 2024-11-19T17:48:27,931 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b5e70f2bbd154bc29a4810e4d5bbe694 2024-11-19T17:48:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/ebb58f7123f641359c388ebc07209351 is 50, key is test_row_0/C:col10/1732038506902/Put/seqid=0 2024-11-19T17:48:27,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741947_1123 (size=12301) 2024-11-19T17:48:27,966 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/ebb58f7123f641359c388ebc07209351 2024-11-19T17:48:27,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/133f6459804a406f8c1a2d351a1bbae5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/133f6459804a406f8c1a2d351a1bbae5 2024-11-19T17:48:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:48:27,984 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/133f6459804a406f8c1a2d351a1bbae5, entries=150, sequenceid=524, filesize=12.0 K 2024-11-19T17:48:27,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/b5e70f2bbd154bc29a4810e4d5bbe694 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b5e70f2bbd154bc29a4810e4d5bbe694 2024-11-19T17:48:27,993 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b5e70f2bbd154bc29a4810e4d5bbe694, entries=150, sequenceid=524, filesize=12.0 K 2024-11-19T17:48:27,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/ebb58f7123f641359c388ebc07209351 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/ebb58f7123f641359c388ebc07209351 2024-11-19T17:48:28,003 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/ebb58f7123f641359c388ebc07209351, entries=150, sequenceid=524, filesize=12.0 K 2024-11-19T17:48:28,005 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 66eb19cac62d93a8797a177e6c688370 in 507ms, sequenceid=524, compaction requested=false 2024-11-19T17:48:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-19T17:48:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-19T17:48:28,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T17:48:28,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1650 sec 2024-11-19T17:48:28,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1710 sec 2024-11-19T17:48:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:28,213 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:28,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T17:48:28,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/51e8e891dc9147088d61b9b1c4038771 is 50, key is test_row_0/A:col10/1732038508212/Put/seqid=0 2024-11-19T17:48:28,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741948_1124 (size=17181) 2024-11-19T17:48:28,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038568245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038568245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038568359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038568359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,482 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T17:48:28,482 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T17:48:28,482 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:28,482 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:28,482 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T17:48:28,482 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:28,482 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T17:48:28,482 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:28,486 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:28,486 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:28,486 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:28,486 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:28,487 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:28,487 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:28,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:28,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:28,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:28,487 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:28,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:28,487 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:28,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038568569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038568569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51452 deadline: 1732038568608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,609 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in 66eb19cac62d93a8797a177e6c688370 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/51e8e891dc9147088d61b9b1c4038771 2024-11-19T17:48:28,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/60978fd646db4394850676510fa6d485 is 50, key is test_row_0/B:col10/1732038508212/Put/seqid=0 2024-11-19T17:48:28,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741949_1125 (size=12301) 2024-11-19T17:48:28,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038568879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038568880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:28,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:48:28,990 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:29,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/60978fd646db4394850676510fa6d485 2024-11-19T17:48:29,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/fd1f941ad560441a8acdbf99ccd6c3c3 is 50, key is test_row_0/C:col10/1732038508212/Put/seqid=0 2024-11-19T17:48:29,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741950_1126 (size=12301) 2024-11-19T17:48:29,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1732038569389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51476 deadline: 1732038569390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:29,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:29,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:29,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:29,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:29,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:29,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:48:29,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/fd1f941ad560441a8acdbf99ccd6c3c3 2024-11-19T17:48:29,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/51e8e891dc9147088d61b9b1c4038771 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/51e8e891dc9147088d61b9b1c4038771 2024-11-19T17:48:29,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/51e8e891dc9147088d61b9b1c4038771, entries=250, sequenceid=543, filesize=16.8 K 2024-11-19T17:48:29,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/60978fd646db4394850676510fa6d485 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/60978fd646db4394850676510fa6d485 2024-11-19T17:48:29,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/60978fd646db4394850676510fa6d485, entries=150, sequenceid=543, filesize=12.0 K 2024-11-19T17:48:29,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/fd1f941ad560441a8acdbf99ccd6c3c3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/fd1f941ad560441a8acdbf99ccd6c3c3 2024-11-19T17:48:29,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/fd1f941ad560441a8acdbf99ccd6c3c3, entries=150, sequenceid=543, filesize=12.0 K 2024-11-19T17:48:29,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 66eb19cac62d93a8797a177e6c688370 in 1293ms, sequenceid=543, compaction requested=true 2024-11-19T17:48:29,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:29,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:29,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:29,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:29,506 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:29,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:29,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66eb19cac62d93a8797a177e6c688370:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:29,506 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:29,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:29,507 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:29,507 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/B is initiating minor compaction (all files) 2024-11-19T17:48:29,507 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:29,507 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/B in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:29,507 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/A is initiating minor compaction (all files) 2024-11-19T17:48:29,507 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/A in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:29,508 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b0eef43b10004969a3d9acba54472b69, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b5e70f2bbd154bc29a4810e4d5bbe694, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/60978fd646db4394850676510fa6d485] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=37.2 K 2024-11-19T17:48:29,508 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/114deef2eaf04eff8dc40b022d8d024f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/133f6459804a406f8c1a2d351a1bbae5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/51e8e891dc9147088d61b9b1c4038771] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=42.0 K 2024-11-19T17:48:29,508 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b0eef43b10004969a3d9acba54472b69, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732038506414 2024-11-19T17:48:29,508 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 114deef2eaf04eff8dc40b022d8d024f, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732038506414 2024-11-19T17:48:29,508 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e70f2bbd154bc29a4810e4d5bbe694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732038506902 2024-11-19T17:48:29,508 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 133f6459804a406f8c1a2d351a1bbae5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732038506902 2024-11-19T17:48:29,509 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 60978fd646db4394850676510fa6d485, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1732038507569 2024-11-19T17:48:29,509 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51e8e891dc9147088d61b9b1c4038771, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1732038507569 2024-11-19T17:48:29,517 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:29,517 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/81ca0964c6e74af4957d91cd576fb83b is 50, key is test_row_0/B:col10/1732038508212/Put/seqid=0 2024-11-19T17:48:29,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741951_1127 (size=13595) 2024-11-19T17:48:29,529 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/81ca0964c6e74af4957d91cd576fb83b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/81ca0964c6e74af4957d91cd576fb83b 2024-11-19T17:48:29,542 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/B of 66eb19cac62d93a8797a177e6c688370 into 81ca0964c6e74af4957d91cd576fb83b(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:29,542 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:29,542 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/B, priority=13, startTime=1732038509506; duration=0sec 2024-11-19T17:48:29,543 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:29,543 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:B 2024-11-19T17:48:29,543 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:29,544 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:29,544 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 66eb19cac62d93a8797a177e6c688370/C is initiating minor compaction (all files) 2024-11-19T17:48:29,544 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66eb19cac62d93a8797a177e6c688370/C in TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:29,544 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/abfa619687844c5b9dd6d082a8960af2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/ebb58f7123f641359c388ebc07209351, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/fd1f941ad560441a8acdbf99ccd6c3c3] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp, totalSize=37.2 K 2024-11-19T17:48:29,545 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting abfa619687844c5b9dd6d082a8960af2, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732038506414 2024-11-19T17:48:29,545 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb58f7123f641359c388ebc07209351, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732038506902 2024-11-19T17:48:29,546 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting fd1f941ad560441a8acdbf99ccd6c3c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1732038507569 2024-11-19T17:48:29,550 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#A#compaction#115 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:29,551 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/2130374199284486bc02770fc5056338 is 50, key is test_row_0/A:col10/1732038508212/Put/seqid=0 2024-11-19T17:48:29,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741952_1128 (size=13595) 2024-11-19T17:48:29,564 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66eb19cac62d93a8797a177e6c688370#C#compaction#116 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:29,565 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/714e5438c4ab4f41986f7f660fe6a410 is 50, key is test_row_0/C:col10/1732038508212/Put/seqid=0 2024-11-19T17:48:29,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741953_1129 (size=13595) 2024-11-19T17:48:29,680 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:48:29,966 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/2130374199284486bc02770fc5056338 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2130374199284486bc02770fc5056338 2024-11-19T17:48:29,980 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/A of 66eb19cac62d93a8797a177e6c688370 into 2130374199284486bc02770fc5056338(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:29,980 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:29,980 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/A, priority=13, startTime=1732038509506; duration=0sec 2024-11-19T17:48:29,980 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:29,980 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:A 2024-11-19T17:48:29,982 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/714e5438c4ab4f41986f7f660fe6a410 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/714e5438c4ab4f41986f7f660fe6a410 2024-11-19T17:48:29,987 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66eb19cac62d93a8797a177e6c688370/C of 66eb19cac62d93a8797a177e6c688370 into 714e5438c4ab4f41986f7f660fe6a410(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:29,987 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:29,987 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., storeName=66eb19cac62d93a8797a177e6c688370/C, priority=13, startTime=1732038509506; duration=0sec 2024-11-19T17:48:29,987 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:29,987 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66eb19cac62d93a8797a177e6c688370:C 2024-11-19T17:48:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:30,410 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:30,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T17:48:30,410 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T17:48:30,410 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T17:48:30,410 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:30,410 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:30,410 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:30,410 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:30,410 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:30,410 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:30,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:30,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:30,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/e8c9bdabd3824915b112acef76358a91 is 50, key is test_row_0/A:col10/1732038508234/Put/seqid=0 2024-11-19T17:48:30,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741954_1130 (size=12301) 2024-11-19T17:48:30,623 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T17:48:30,623 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:30,623 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:30,624 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:30,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:30,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/e8c9bdabd3824915b112acef76358a91 2024-11-19T17:48:30,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/5e8f808f2e254dc18d60d48ad241f833 is 50, key is test_row_0/B:col10/1732038508234/Put/seqid=0 2024-11-19T17:48:30,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741955_1131 (size=12301) 2024-11-19T17:48:31,210 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T17:48:31,210 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:31,210 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:31,210 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:31,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:31,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/5e8f808f2e254dc18d60d48ad241f833 2024-11-19T17:48:31,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/19e9ead173d34c1cbb97a90c8738993f is 50, key is test_row_0/C:col10/1732038508234/Put/seqid=0 2024-11-19T17:48:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741956_1132 (size=12301) 2024-11-19T17:48:31,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=568 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/19e9ead173d34c1cbb97a90c8738993f 2024-11-19T17:48:31,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/e8c9bdabd3824915b112acef76358a91 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e8c9bdabd3824915b112acef76358a91 2024-11-19T17:48:31,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e8c9bdabd3824915b112acef76358a91, entries=150, sequenceid=568, filesize=12.0 K 2024-11-19T17:48:31,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/5e8f808f2e254dc18d60d48ad241f833 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/5e8f808f2e254dc18d60d48ad241f833 2024-11-19T17:48:31,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/5e8f808f2e254dc18d60d48ad241f833, entries=150, sequenceid=568, filesize=12.0 K 2024-11-19T17:48:31,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/19e9ead173d34c1cbb97a90c8738993f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/19e9ead173d34c1cbb97a90c8738993f 2024-11-19T17:48:31,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/19e9ead173d34c1cbb97a90c8738993f, entries=150, sequenceid=568, filesize=12.0 K 2024-11-19T17:48:31,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=13.42 KB/13740 for 66eb19cac62d93a8797a177e6c688370 in 1261ms, sequenceid=568, compaction requested=false 2024-11-19T17:48:31,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66eb19cac62d93a8797a177e6c688370: 2024-11-19T17:48:37,335 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T17:48:37,335 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:48:37,335 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:37,336 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:37,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:37,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:48:37,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-19T17:48:37,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-19T17:48:37,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-19T17:48:37,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-19T17:48:37,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-19T17:48:37,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:48:37,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3767 2024-11-19T17:48:37,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3590 2024-11-19T17:48:37,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:48:37,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1650 2024-11-19T17:48:37,339 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4950 rows 2024-11-19T17:48:37,339 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1692 2024-11-19T17:48:37,339 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5066 rows 2024-11-19T17:48:37,340 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:48:37,340 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:48:37,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:37,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:37,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:48:37,344 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:48:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:48:37,353 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038517353"}]},"ts":"1732038517353"} 2024-11-19T17:48:37,355 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:48:37,355 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:48:37,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:48:37,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, UNASSIGN}] 2024-11-19T17:48:37,364 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, UNASSIGN 2024-11-19T17:48:37,365 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=66eb19cac62d93a8797a177e6c688370, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:37,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, UNASSIGN because future has completed 2024-11-19T17:48:37,368 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:48:37,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:48:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:48:37,524 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:37,525 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:48:37,525 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 66eb19cac62d93a8797a177e6c688370, disabling compactions & flushes 2024-11-19T17:48:37,525 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:37,525 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:37,525 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. after waiting 0 ms 2024-11-19T17:48:37,525 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:37,525 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 66eb19cac62d93a8797a177e6c688370 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-19T17:48:37,529 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/5ce19eb2b2cc4748b235271ceadaf62a is 50, key is test_row_0/A:col10/1732038517330/Put/seqid=0 2024-11-19T17:48:37,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741957_1133 (size=9857) 2024-11-19T17:48:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:48:37,934 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/5ce19eb2b2cc4748b235271ceadaf62a 2024-11-19T17:48:37,941 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/a0e1587b9dc642e78a1ab4da76fcdc1b is 50, key is test_row_0/B:col10/1732038517330/Put/seqid=0 2024-11-19T17:48:37,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741958_1134 (size=9857) 2024-11-19T17:48:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:48:38,345 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/a0e1587b9dc642e78a1ab4da76fcdc1b 2024-11-19T17:48:38,352 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/15a5c59be1c14368aec99efc13715804 is 50, key is test_row_0/C:col10/1732038517330/Put/seqid=0 2024-11-19T17:48:38,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741959_1135 (size=9857) 2024-11-19T17:48:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:48:38,757 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/15a5c59be1c14368aec99efc13715804 2024-11-19T17:48:38,762 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/A/5ce19eb2b2cc4748b235271ceadaf62a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5ce19eb2b2cc4748b235271ceadaf62a 2024-11-19T17:48:38,766 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5ce19eb2b2cc4748b235271ceadaf62a, entries=100, sequenceid=574, filesize=9.6 K 2024-11-19T17:48:38,767 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/B/a0e1587b9dc642e78a1ab4da76fcdc1b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a0e1587b9dc642e78a1ab4da76fcdc1b 2024-11-19T17:48:38,771 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a0e1587b9dc642e78a1ab4da76fcdc1b, entries=100, sequenceid=574, filesize=9.6 K 2024-11-19T17:48:38,772 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/.tmp/C/15a5c59be1c14368aec99efc13715804 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/15a5c59be1c14368aec99efc13715804 2024-11-19T17:48:38,775 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/15a5c59be1c14368aec99efc13715804, entries=100, sequenceid=574, filesize=9.6 K 2024-11-19T17:48:38,776 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 66eb19cac62d93a8797a177e6c688370 in 1251ms, sequenceid=574, compaction requested=true 2024-11-19T17:48:38,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c967ecc3bbf04f44b987044553360e17, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ed4c2b19cd8a4535ba07bcabf0850b05, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/f982baddf25d40a195ae34caee9d567e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8a23c4ebe0af422196bfff3b5710cd21, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b9f164224c4f4deda4630bd2d74fc011, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c0b719f9a541410da732bfee371d3e36, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5e8de40c562e410babc8fd83d7d9b421, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/41a45b55493f4467965a05215febe91c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2a89620bf15a4fa7a73f98a91f9dacb1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5d9c79e2fee44f6813242c996cee562, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/4c5346103911491786a44c07af9d26b8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/56f4fbcb828d43e69a24d6b938efac3c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0c06c3c93594465a825408263956457a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5881c021bab430b95cdd9db29077a82, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/335e5953c9d64ebb92fd0532ae390ffc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/247a4bff44084704af7bbed1b7f06705, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c006a3827ccc4b6880feb6ba68a30270, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0854f9edf5b74beba74eafbd8c1f60af, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ffc34ec4cabf41ef8f25ad806aa605ba, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1915428845454b53add056a9f231d994, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/abe883833494447287c2bf743243fd77, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/afb88348c9004347b75a47e41eb7b518, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ccbb8e49ce20457caa99a0877f00a21c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/62c1d766b8fa454280665a388f6d8b78, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0b1b5c8c2163469abb381d4d9d1d8366, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/620aa1b2b6944a629981cb7cb50c8af8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/63ba20eac9c44e88bee436c7a3c0f3ec, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/58cb72e3d051490683b7c185f28b1267, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/9e00b5843ef04ab3853c4e99908c4e16, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1e380271febc40a2b15779c74c3f4bf5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/352ef66eb5924883b55eb020e40a8ea5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0f5d42af0dc34786b38220b73e2d2d22, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/fa7cd9ecb9f94241b1bcf89d1fd5ce35, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e2507a948cf248a3b40eb137da1d73e7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/114deef2eaf04eff8dc40b022d8d024f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8fb911675eab4891b998b3cf7f38b499, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/133f6459804a406f8c1a2d351a1bbae5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/51e8e891dc9147088d61b9b1c4038771] to archive 2024-11-19T17:48:38,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:48:38,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c967ecc3bbf04f44b987044553360e17 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c967ecc3bbf04f44b987044553360e17 2024-11-19T17:48:38,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ed4c2b19cd8a4535ba07bcabf0850b05 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ed4c2b19cd8a4535ba07bcabf0850b05 2024-11-19T17:48:38,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/f982baddf25d40a195ae34caee9d567e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/f982baddf25d40a195ae34caee9d567e 2024-11-19T17:48:38,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8a23c4ebe0af422196bfff3b5710cd21 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8a23c4ebe0af422196bfff3b5710cd21 2024-11-19T17:48:38,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b9f164224c4f4deda4630bd2d74fc011 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b9f164224c4f4deda4630bd2d74fc011 2024-11-19T17:48:38,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c0b719f9a541410da732bfee371d3e36 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c0b719f9a541410da732bfee371d3e36 2024-11-19T17:48:38,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5e8de40c562e410babc8fd83d7d9b421 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5e8de40c562e410babc8fd83d7d9b421 2024-11-19T17:48:38,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/41a45b55493f4467965a05215febe91c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/41a45b55493f4467965a05215febe91c 2024-11-19T17:48:38,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2a89620bf15a4fa7a73f98a91f9dacb1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2a89620bf15a4fa7a73f98a91f9dacb1 2024-11-19T17:48:38,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5d9c79e2fee44f6813242c996cee562 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5d9c79e2fee44f6813242c996cee562 2024-11-19T17:48:38,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/4c5346103911491786a44c07af9d26b8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/4c5346103911491786a44c07af9d26b8 2024-11-19T17:48:38,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/56f4fbcb828d43e69a24d6b938efac3c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/56f4fbcb828d43e69a24d6b938efac3c 2024-11-19T17:48:38,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0c06c3c93594465a825408263956457a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0c06c3c93594465a825408263956457a 2024-11-19T17:48:38,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5881c021bab430b95cdd9db29077a82 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/b5881c021bab430b95cdd9db29077a82 2024-11-19T17:48:38,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/335e5953c9d64ebb92fd0532ae390ffc to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/335e5953c9d64ebb92fd0532ae390ffc 2024-11-19T17:48:38,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/247a4bff44084704af7bbed1b7f06705 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/247a4bff44084704af7bbed1b7f06705 2024-11-19T17:48:38,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c006a3827ccc4b6880feb6ba68a30270 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/c006a3827ccc4b6880feb6ba68a30270 2024-11-19T17:48:38,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0854f9edf5b74beba74eafbd8c1f60af to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0854f9edf5b74beba74eafbd8c1f60af 2024-11-19T17:48:38,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ffc34ec4cabf41ef8f25ad806aa605ba to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ffc34ec4cabf41ef8f25ad806aa605ba 2024-11-19T17:48:38,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1915428845454b53add056a9f231d994 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1915428845454b53add056a9f231d994 2024-11-19T17:48:38,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/abe883833494447287c2bf743243fd77 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/abe883833494447287c2bf743243fd77 2024-11-19T17:48:38,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/afb88348c9004347b75a47e41eb7b518 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/afb88348c9004347b75a47e41eb7b518 2024-11-19T17:48:38,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ccbb8e49ce20457caa99a0877f00a21c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/ccbb8e49ce20457caa99a0877f00a21c 2024-11-19T17:48:38,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/62c1d766b8fa454280665a388f6d8b78 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/62c1d766b8fa454280665a388f6d8b78 2024-11-19T17:48:38,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0b1b5c8c2163469abb381d4d9d1d8366 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0b1b5c8c2163469abb381d4d9d1d8366 2024-11-19T17:48:38,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/620aa1b2b6944a629981cb7cb50c8af8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/620aa1b2b6944a629981cb7cb50c8af8 2024-11-19T17:48:38,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/63ba20eac9c44e88bee436c7a3c0f3ec to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/63ba20eac9c44e88bee436c7a3c0f3ec 2024-11-19T17:48:38,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/58cb72e3d051490683b7c185f28b1267 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/58cb72e3d051490683b7c185f28b1267 2024-11-19T17:48:38,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/9e00b5843ef04ab3853c4e99908c4e16 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/9e00b5843ef04ab3853c4e99908c4e16 2024-11-19T17:48:38,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1e380271febc40a2b15779c74c3f4bf5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/1e380271febc40a2b15779c74c3f4bf5 2024-11-19T17:48:38,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/352ef66eb5924883b55eb020e40a8ea5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/352ef66eb5924883b55eb020e40a8ea5 2024-11-19T17:48:38,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0f5d42af0dc34786b38220b73e2d2d22 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/0f5d42af0dc34786b38220b73e2d2d22 2024-11-19T17:48:38,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/fa7cd9ecb9f94241b1bcf89d1fd5ce35 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/fa7cd9ecb9f94241b1bcf89d1fd5ce35 2024-11-19T17:48:38,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e2507a948cf248a3b40eb137da1d73e7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e2507a948cf248a3b40eb137da1d73e7 2024-11-19T17:48:38,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/114deef2eaf04eff8dc40b022d8d024f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/114deef2eaf04eff8dc40b022d8d024f 2024-11-19T17:48:38,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8fb911675eab4891b998b3cf7f38b499 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/8fb911675eab4891b998b3cf7f38b499 2024-11-19T17:48:38,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/133f6459804a406f8c1a2d351a1bbae5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/133f6459804a406f8c1a2d351a1bbae5 2024-11-19T17:48:38,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/51e8e891dc9147088d61b9b1c4038771 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/51e8e891dc9147088d61b9b1c4038771 2024-11-19T17:48:38,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/d889309c12e94313988f09657ab5d066, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/77d4b6ac0f104fcc9e2e276e958be231, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/45f4d5c2245c458297e72f5dba3b3834, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/63db62db8adc48409a9291c43a30e7a5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/984b34cc73944f4c8ecde0f8f4b4bb38, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/68fb871e105246e5b7495f549aae8489, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6468e79fe60747feaefddea97a7e7498, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4073a60483b343c1b3852d3137ca0b60, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ff0e645a59d45c4ae934b9b30498622, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/8d2a3e43461e438890adb9e8af665127, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/416610d2c2ac4ffab6b4fd1ddf28dc7d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/bfe1dff327f04e51984bfef2e257f2f4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e022099490bf403e99b11ed013585116, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/df869f30a7ee415d852a71044b5e512a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a14fc01607184bf080ce74ea1c73e301, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/db04c30d931b44c3913e9cd17209661b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1a0f04e6f97140d28368d172f5fdfeaa, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6c5c978b9ee14ae783a5d3d82c2c54ea, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a1f080b5b0cf4021b52fb909386cb075, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4ca7bd79ae674f98b003df0465863e80, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e48b97a47ed04d7ca5480194ecb8eb36, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b4caf318826e4f869c338e7f7b834b60, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4474865e306745fe92640ff8e4df25f0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1eec565377bd451bbfaa0bad8f82c2f8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/818851b1dceb4a0abee38f1714f4a476, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/496881be4d4542fa8a78d65f220a3705, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ca557c693714c22b224eb7545bc2a95, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/c662b9ee4f6a40b48f36d9d14b3a9a36, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0e93ff8ba2814469b55785f7f53a24ce, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/20e30f6acf4145af8ff113efc18a189b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/cf3dd564e5ab4a5690915a0fc91192a7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/3edfed4c5dda4079858c211f175b40e3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0c04391185ef426e94762285ebcb1d15, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b19b30054ea84bf7ac5fbbf8faeaae6f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b0eef43b10004969a3d9acba54472b69, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/f2c80dfd0ba0461aadbdde9974d868fc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b5e70f2bbd154bc29a4810e4d5bbe694, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/60978fd646db4394850676510fa6d485] to archive 2024-11-19T17:48:38,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:48:38,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/d889309c12e94313988f09657ab5d066 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/d889309c12e94313988f09657ab5d066 2024-11-19T17:48:38,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/77d4b6ac0f104fcc9e2e276e958be231 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/77d4b6ac0f104fcc9e2e276e958be231 2024-11-19T17:48:38,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/45f4d5c2245c458297e72f5dba3b3834 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/45f4d5c2245c458297e72f5dba3b3834 2024-11-19T17:48:38,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/63db62db8adc48409a9291c43a30e7a5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/63db62db8adc48409a9291c43a30e7a5 2024-11-19T17:48:38,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/984b34cc73944f4c8ecde0f8f4b4bb38 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/984b34cc73944f4c8ecde0f8f4b4bb38 2024-11-19T17:48:38,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/68fb871e105246e5b7495f549aae8489 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/68fb871e105246e5b7495f549aae8489 2024-11-19T17:48:38,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6468e79fe60747feaefddea97a7e7498 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6468e79fe60747feaefddea97a7e7498 2024-11-19T17:48:38,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4073a60483b343c1b3852d3137ca0b60 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4073a60483b343c1b3852d3137ca0b60 2024-11-19T17:48:38,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ff0e645a59d45c4ae934b9b30498622 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ff0e645a59d45c4ae934b9b30498622 2024-11-19T17:48:38,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/8d2a3e43461e438890adb9e8af665127 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/8d2a3e43461e438890adb9e8af665127 2024-11-19T17:48:38,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/416610d2c2ac4ffab6b4fd1ddf28dc7d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/416610d2c2ac4ffab6b4fd1ddf28dc7d 2024-11-19T17:48:38,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/bfe1dff327f04e51984bfef2e257f2f4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/bfe1dff327f04e51984bfef2e257f2f4 2024-11-19T17:48:38,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e022099490bf403e99b11ed013585116 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e022099490bf403e99b11ed013585116 2024-11-19T17:48:38,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/df869f30a7ee415d852a71044b5e512a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/df869f30a7ee415d852a71044b5e512a 2024-11-19T17:48:38,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a14fc01607184bf080ce74ea1c73e301 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a14fc01607184bf080ce74ea1c73e301 2024-11-19T17:48:38,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/db04c30d931b44c3913e9cd17209661b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/db04c30d931b44c3913e9cd17209661b 2024-11-19T17:48:38,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1a0f04e6f97140d28368d172f5fdfeaa to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1a0f04e6f97140d28368d172f5fdfeaa 2024-11-19T17:48:38,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6c5c978b9ee14ae783a5d3d82c2c54ea to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/6c5c978b9ee14ae783a5d3d82c2c54ea 2024-11-19T17:48:38,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a1f080b5b0cf4021b52fb909386cb075 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a1f080b5b0cf4021b52fb909386cb075 2024-11-19T17:48:38,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4ca7bd79ae674f98b003df0465863e80 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4ca7bd79ae674f98b003df0465863e80 2024-11-19T17:48:38,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e48b97a47ed04d7ca5480194ecb8eb36 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/e48b97a47ed04d7ca5480194ecb8eb36 2024-11-19T17:48:38,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b4caf318826e4f869c338e7f7b834b60 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b4caf318826e4f869c338e7f7b834b60 2024-11-19T17:48:38,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4474865e306745fe92640ff8e4df25f0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/4474865e306745fe92640ff8e4df25f0 2024-11-19T17:48:38,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1eec565377bd451bbfaa0bad8f82c2f8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1eec565377bd451bbfaa0bad8f82c2f8 2024-11-19T17:48:38,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/818851b1dceb4a0abee38f1714f4a476 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/818851b1dceb4a0abee38f1714f4a476 2024-11-19T17:48:38,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/496881be4d4542fa8a78d65f220a3705 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/496881be4d4542fa8a78d65f220a3705 2024-11-19T17:48:38,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ca557c693714c22b224eb7545bc2a95 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/1ca557c693714c22b224eb7545bc2a95 2024-11-19T17:48:38,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/c662b9ee4f6a40b48f36d9d14b3a9a36 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/c662b9ee4f6a40b48f36d9d14b3a9a36 2024-11-19T17:48:38,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0e93ff8ba2814469b55785f7f53a24ce to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0e93ff8ba2814469b55785f7f53a24ce 2024-11-19T17:48:38,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/20e30f6acf4145af8ff113efc18a189b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/20e30f6acf4145af8ff113efc18a189b 2024-11-19T17:48:38,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/cf3dd564e5ab4a5690915a0fc91192a7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/cf3dd564e5ab4a5690915a0fc91192a7 2024-11-19T17:48:38,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/3edfed4c5dda4079858c211f175b40e3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/3edfed4c5dda4079858c211f175b40e3 2024-11-19T17:48:38,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0c04391185ef426e94762285ebcb1d15 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/0c04391185ef426e94762285ebcb1d15 2024-11-19T17:48:38,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b19b30054ea84bf7ac5fbbf8faeaae6f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b19b30054ea84bf7ac5fbbf8faeaae6f 2024-11-19T17:48:38,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b0eef43b10004969a3d9acba54472b69 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b0eef43b10004969a3d9acba54472b69 2024-11-19T17:48:38,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/f2c80dfd0ba0461aadbdde9974d868fc to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/f2c80dfd0ba0461aadbdde9974d868fc 2024-11-19T17:48:38,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b5e70f2bbd154bc29a4810e4d5bbe694 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/b5e70f2bbd154bc29a4810e4d5bbe694 2024-11-19T17:48:38,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/60978fd646db4394850676510fa6d485 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/60978fd646db4394850676510fa6d485 2024-11-19T17:48:38,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b66ff29003d647efa7fa00cc93f1053a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/a918ca280b1543af8baca0c99fa94b41, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/4b3a34a1d39345ddaf0ee24bca2e2f66, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6c40eeb6445948d4a5ac812d223c1d3c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/cb01b3e453974ed38370dfbb37ebe104, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9b11176eb42d454a935e347e5755fe54, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/61562f682d7047a4a076ed80aa41cbba, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8d50e67a273241c7b29104bd930ae0b9, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9298641653014f76bbb476c88167efb8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/97818f4a77fe4779bbfd3b0f664fae17, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9675dfa8978e4dfe8306e9edfec6708b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/53be3535b742423c92d4f5c12b6dffcb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/c238ca4355544c3db477f4cca85ed84a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/d8fe91ffb48e4925b4f2f77915f81b53, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/905bc21c70454136972b822a4763b584, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/7257f8789f5e4162bd1b209ca5aa95bf, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d01ad6643944d8999df0fe9e7e9ede6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/54947c680a5b4d63b1e3eb2b979eaa04, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b457c8cfd539456e901909bb134fe845, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9fcc7b30a8f24e9d9e74851651e39289, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/59b7b36ea3674c49a7c856a6154220c2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/86f8ab936a47416e8fc8629a3f24829c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/bfe1993830874b3e9f54c44622f1b64e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8ef6304de1d2427a9173f049f015f50d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/599ffaf0f5c74951a3396b7a7cd5ad3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6883b51446a3405dadb7c1e846552309, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8595c0c2723c4f76b88f2d3f2198736f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/2e4626d35b5f401ab1d845aeb1cda3de, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/5243f919f2094514b13f09d63a83025d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/916dfc40675744b890964bc002cc7826, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d061808f10d4ea0821948fe90e054dd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/77282042f3774589bb359f183ce464ee, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/618a3b21ba4d40abba5e46343832ac64, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/918625448004493898923f0866911970, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/abfa619687844c5b9dd6d082a8960af2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/834937c620024d2da419865a818a2545, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/ebb58f7123f641359c388ebc07209351, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/fd1f941ad560441a8acdbf99ccd6c3c3] to archive 2024-11-19T17:48:38,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:48:38,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b66ff29003d647efa7fa00cc93f1053a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b66ff29003d647efa7fa00cc93f1053a 2024-11-19T17:48:38,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/a918ca280b1543af8baca0c99fa94b41 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/a918ca280b1543af8baca0c99fa94b41 2024-11-19T17:48:38,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/4b3a34a1d39345ddaf0ee24bca2e2f66 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/4b3a34a1d39345ddaf0ee24bca2e2f66 2024-11-19T17:48:38,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6c40eeb6445948d4a5ac812d223c1d3c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6c40eeb6445948d4a5ac812d223c1d3c 2024-11-19T17:48:38,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/cb01b3e453974ed38370dfbb37ebe104 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/cb01b3e453974ed38370dfbb37ebe104 2024-11-19T17:48:38,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9b11176eb42d454a935e347e5755fe54 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9b11176eb42d454a935e347e5755fe54 2024-11-19T17:48:38,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/61562f682d7047a4a076ed80aa41cbba to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/61562f682d7047a4a076ed80aa41cbba 2024-11-19T17:48:38,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8d50e67a273241c7b29104bd930ae0b9 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8d50e67a273241c7b29104bd930ae0b9 2024-11-19T17:48:38,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9298641653014f76bbb476c88167efb8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9298641653014f76bbb476c88167efb8 2024-11-19T17:48:38,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/97818f4a77fe4779bbfd3b0f664fae17 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/97818f4a77fe4779bbfd3b0f664fae17 2024-11-19T17:48:38,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9675dfa8978e4dfe8306e9edfec6708b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9675dfa8978e4dfe8306e9edfec6708b 2024-11-19T17:48:38,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/53be3535b742423c92d4f5c12b6dffcb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/53be3535b742423c92d4f5c12b6dffcb 2024-11-19T17:48:38,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/c238ca4355544c3db477f4cca85ed84a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/c238ca4355544c3db477f4cca85ed84a 2024-11-19T17:48:38,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/d8fe91ffb48e4925b4f2f77915f81b53 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/d8fe91ffb48e4925b4f2f77915f81b53 2024-11-19T17:48:38,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/905bc21c70454136972b822a4763b584 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/905bc21c70454136972b822a4763b584 2024-11-19T17:48:38,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/7257f8789f5e4162bd1b209ca5aa95bf to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/7257f8789f5e4162bd1b209ca5aa95bf 2024-11-19T17:48:38,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d01ad6643944d8999df0fe9e7e9ede6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d01ad6643944d8999df0fe9e7e9ede6 2024-11-19T17:48:38,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/54947c680a5b4d63b1e3eb2b979eaa04 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/54947c680a5b4d63b1e3eb2b979eaa04 2024-11-19T17:48:38,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b457c8cfd539456e901909bb134fe845 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/b457c8cfd539456e901909bb134fe845 2024-11-19T17:48:38,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9fcc7b30a8f24e9d9e74851651e39289 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/9fcc7b30a8f24e9d9e74851651e39289 2024-11-19T17:48:38,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/59b7b36ea3674c49a7c856a6154220c2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/59b7b36ea3674c49a7c856a6154220c2 2024-11-19T17:48:38,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/86f8ab936a47416e8fc8629a3f24829c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/86f8ab936a47416e8fc8629a3f24829c 2024-11-19T17:48:38,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/bfe1993830874b3e9f54c44622f1b64e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/bfe1993830874b3e9f54c44622f1b64e 2024-11-19T17:48:38,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8ef6304de1d2427a9173f049f015f50d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8ef6304de1d2427a9173f049f015f50d 2024-11-19T17:48:38,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/599ffaf0f5c74951a3396b7a7cd5ad3d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/599ffaf0f5c74951a3396b7a7cd5ad3d 2024-11-19T17:48:38,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6883b51446a3405dadb7c1e846552309 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/6883b51446a3405dadb7c1e846552309 2024-11-19T17:48:38,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8595c0c2723c4f76b88f2d3f2198736f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/8595c0c2723c4f76b88f2d3f2198736f 2024-11-19T17:48:38,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/2e4626d35b5f401ab1d845aeb1cda3de to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/2e4626d35b5f401ab1d845aeb1cda3de 2024-11-19T17:48:38,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/5243f919f2094514b13f09d63a83025d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/5243f919f2094514b13f09d63a83025d 2024-11-19T17:48:38,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/916dfc40675744b890964bc002cc7826 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/916dfc40675744b890964bc002cc7826 2024-11-19T17:48:38,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d061808f10d4ea0821948fe90e054dd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/3d061808f10d4ea0821948fe90e054dd 2024-11-19T17:48:38,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/77282042f3774589bb359f183ce464ee to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/77282042f3774589bb359f183ce464ee 2024-11-19T17:48:38,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/618a3b21ba4d40abba5e46343832ac64 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/618a3b21ba4d40abba5e46343832ac64 2024-11-19T17:48:38,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/918625448004493898923f0866911970 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/918625448004493898923f0866911970 2024-11-19T17:48:38,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/abfa619687844c5b9dd6d082a8960af2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/abfa619687844c5b9dd6d082a8960af2 2024-11-19T17:48:38,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/834937c620024d2da419865a818a2545 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/834937c620024d2da419865a818a2545 2024-11-19T17:48:38,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/ebb58f7123f641359c388ebc07209351 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/ebb58f7123f641359c388ebc07209351 2024-11-19T17:48:38,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/fd1f941ad560441a8acdbf99ccd6c3c3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/fd1f941ad560441a8acdbf99ccd6c3c3 2024-11-19T17:48:38,927 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/recovered.edits/577.seqid, newMaxSeqId=577, maxSeqId=1 2024-11-19T17:48:38,927 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. 2024-11-19T17:48:38,928 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 66eb19cac62d93a8797a177e6c688370: Waiting for close lock at 1732038517525Running coprocessor pre-close hooks at 1732038517525Disabling compacts and flushes for region at 1732038517525Disabling writes for close at 1732038517525Obtaining lock to block concurrent updates at 1732038517525Preparing flush snapshotting stores in 66eb19cac62d93a8797a177e6c688370 at 1732038517525Finished memstore snapshotting TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732038517526 (+1 ms)Flushing stores of TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370. at 1732038517526Flushing 66eb19cac62d93a8797a177e6c688370/A: creating writer at 1732038517526Flushing 66eb19cac62d93a8797a177e6c688370/A: appending metadata at 1732038517529 (+3 ms)Flushing 66eb19cac62d93a8797a177e6c688370/A: closing flushed file at 1732038517529Flushing 66eb19cac62d93a8797a177e6c688370/B: creating writer at 1732038517938 (+409 ms)Flushing 66eb19cac62d93a8797a177e6c688370/B: appending metadata at 1732038517940 (+2 ms)Flushing 66eb19cac62d93a8797a177e6c688370/B: closing flushed file at 1732038517940Flushing 66eb19cac62d93a8797a177e6c688370/C: creating writer at 1732038518349 (+409 ms)Flushing 66eb19cac62d93a8797a177e6c688370/C: appending metadata at 1732038518352 (+3 ms)Flushing 66eb19cac62d93a8797a177e6c688370/C: closing flushed file at 1732038518352Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c3958af: reopening flushed file at 1732038518761 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c1bac1f: reopening flushed file at 1732038518766 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b3e55b: reopening flushed file at 1732038518771 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 66eb19cac62d93a8797a177e6c688370 in 1251ms, sequenceid=574, compaction requested=true at 1732038518776 (+5 ms)Writing region close event to WAL at 1732038518923 (+147 ms)Running coprocessor post-close hooks at 1732038518927 (+4 ms)Closed at 1732038518927 2024-11-19T17:48:38,930 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:38,931 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=66eb19cac62d93a8797a177e6c688370, regionState=CLOSED 2024-11-19T17:48:38,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:48:38,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T17:48:38,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 66eb19cac62d93a8797a177e6c688370, server=26d3106abcfe,36839,1732038481522 in 1.5650 sec 2024-11-19T17:48:38,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-19T17:48:38,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66eb19cac62d93a8797a177e6c688370, UNASSIGN in 1.5720 sec 2024-11-19T17:48:38,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T17:48:38,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5800 sec 2024-11-19T17:48:38,940 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038518940"}]},"ts":"1732038518940"} 2024-11-19T17:48:38,941 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:48:38,941 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:48:38,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5960 sec 2024-11-19T17:48:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:48:39,500 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:39,506 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:48:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:39,514 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T17:48:39,516 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:39,520 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:39,524 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/recovered.edits] 2024-11-19T17:48:39,527 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2130374199284486bc02770fc5056338 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/2130374199284486bc02770fc5056338 2024-11-19T17:48:39,529 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5ce19eb2b2cc4748b235271ceadaf62a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/5ce19eb2b2cc4748b235271ceadaf62a 2024-11-19T17:48:39,530 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e8c9bdabd3824915b112acef76358a91 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/A/e8c9bdabd3824915b112acef76358a91 2024-11-19T17:48:39,532 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/5e8f808f2e254dc18d60d48ad241f833 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/5e8f808f2e254dc18d60d48ad241f833 2024-11-19T17:48:39,534 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/81ca0964c6e74af4957d91cd576fb83b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/81ca0964c6e74af4957d91cd576fb83b 2024-11-19T17:48:39,535 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a0e1587b9dc642e78a1ab4da76fcdc1b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/B/a0e1587b9dc642e78a1ab4da76fcdc1b 2024-11-19T17:48:39,538 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/15a5c59be1c14368aec99efc13715804 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/15a5c59be1c14368aec99efc13715804 2024-11-19T17:48:39,539 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/19e9ead173d34c1cbb97a90c8738993f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/19e9ead173d34c1cbb97a90c8738993f 2024-11-19T17:48:39,540 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/714e5438c4ab4f41986f7f660fe6a410 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/C/714e5438c4ab4f41986f7f660fe6a410 2024-11-19T17:48:39,543 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/recovered.edits/577.seqid to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370/recovered.edits/577.seqid 2024-11-19T17:48:39,544 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/66eb19cac62d93a8797a177e6c688370 2024-11-19T17:48:39,544 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:48:39,548 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:39,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T17:48:39,555 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:48:39,579 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:48:39,581 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:39,581 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:48:39,581 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732038519581"}]},"ts":"9223372036854775807"} 2024-11-19T17:48:39,585 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:48:39,585 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 66eb19cac62d93a8797a177e6c688370, NAME => 'TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:48:39,585 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:48:39,585 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732038519585"}]},"ts":"9223372036854775807"} 2024-11-19T17:48:39,588 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:48:39,589 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:39,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-11-19T17:48:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T17:48:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:48:39,619 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:39,631 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1036582635_22 at /127.0.0.1:41730 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;26d3106abcfe:36839-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=708 (was 931), ProcessCount=11 (was 11), AvailableMemoryMB=9724 (was 10482) 2024-11-19T17:48:39,640 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=708, ProcessCount=11, AvailableMemoryMB=9722 2024-11-19T17:48:39,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:48:39,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:48:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:39,644 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:48:39,644 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:39,644 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-19T17:48:39,645 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:48:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T17:48:39,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741960_1136 (size=959) 2024-11-19T17:48:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T17:48:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T17:48:40,055 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d 2024-11-19T17:48:40,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741961_1137 (size=53) 2024-11-19T17:48:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T17:48:40,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:40,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 42eb2452c7f2a30803f85b3a18651fb5, disabling compactions & flushes 2024-11-19T17:48:40,467 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:40,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:40,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. after waiting 0 ms 2024-11-19T17:48:40,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:40,467 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:40,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 42eb2452c7f2a30803f85b3a18651fb5: Waiting for close lock at 1732038520467Disabling compacts and flushes for region at 1732038520467Disabling writes for close at 1732038520467Writing region close event to WAL at 1732038520467Closed at 1732038520467 2024-11-19T17:48:40,468 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:48:40,469 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732038520468"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732038520468"}]},"ts":"1732038520468"} 2024-11-19T17:48:40,470 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:48:40,471 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:48:40,472 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038520471"}]},"ts":"1732038520471"} 2024-11-19T17:48:40,473 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:48:40,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, ASSIGN}] 2024-11-19T17:48:40,474 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, ASSIGN 2024-11-19T17:48:40,475 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, ASSIGN; state=OFFLINE, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=false 2024-11-19T17:48:40,626 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=42eb2452c7f2a30803f85b3a18651fb5, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:40,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, ASSIGN because future has completed 2024-11-19T17:48:40,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:48:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T17:48:40,787 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:40,787 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:48:40,787 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,788 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:40,788 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,788 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,789 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,791 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42eb2452c7f2a30803f85b3a18651fb5 columnFamilyName A 2024-11-19T17:48:40,791 DEBUG [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:40,791 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(327): Store=42eb2452c7f2a30803f85b3a18651fb5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:40,791 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,793 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42eb2452c7f2a30803f85b3a18651fb5 columnFamilyName B 2024-11-19T17:48:40,793 DEBUG [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:40,793 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(327): Store=42eb2452c7f2a30803f85b3a18651fb5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:40,793 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,794 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42eb2452c7f2a30803f85b3a18651fb5 columnFamilyName C 2024-11-19T17:48:40,794 DEBUG [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:40,794 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(327): Store=42eb2452c7f2a30803f85b3a18651fb5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:40,795 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,795 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,795 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,796 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,797 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,797 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:48:40,798 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,800 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:48:40,800 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 42eb2452c7f2a30803f85b3a18651fb5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70313894, jitterRate=0.047758668661117554}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:48:40,801 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:40,801 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 42eb2452c7f2a30803f85b3a18651fb5: Running coprocessor pre-open hook at 1732038520788Writing region info on filesystem at 1732038520788Initializing all the Stores at 1732038520788Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038520789 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038520789Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038520789Cleaning up temporary data from old regions at 1732038520797 (+8 ms)Running coprocessor post-open hooks at 1732038520801 (+4 ms)Region opened successfully at 1732038520801 2024-11-19T17:48:40,802 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., pid=34, masterSystemTime=1732038520784 2024-11-19T17:48:40,804 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:40,804 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:40,805 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=42eb2452c7f2a30803f85b3a18651fb5, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:40,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:48:40,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T17:48:40,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 in 177 msec 2024-11-19T17:48:40,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-19T17:48:40,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, ASSIGN in 337 msec 2024-11-19T17:48:40,813 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:48:40,813 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038520813"}]},"ts":"1732038520813"} 2024-11-19T17:48:40,814 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:48:40,815 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:48:40,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-11-19T17:48:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T17:48:41,791 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:41,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:41,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:41,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:41,797 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:41,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:41,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d85c74e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:41,800 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bdc2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:41,801 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:41,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:41,806 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:48:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,41481,1732038480627 2024-11-19T17:48:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47569338 2024-11-19T17:48:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:48:41,808 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:48:41,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:48:41,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:48:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T17:48:41,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741962_1138 (size=995) 2024-11-19T17:48:42,238 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T17:48:42,238 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T17:48:42,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:48:42,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, REOPEN/MOVE}] 2024-11-19T17:48:42,249 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, REOPEN/MOVE 2024-11-19T17:48:42,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=42eb2452c7f2a30803f85b3a18651fb5, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:42,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, REOPEN/MOVE because future has completed 2024-11-19T17:48:42,252 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:48:42,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:48:42,405 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,405 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:48:42,405 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 42eb2452c7f2a30803f85b3a18651fb5, disabling compactions & flushes 2024-11-19T17:48:42,405 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,405 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,405 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. after waiting 0 ms 2024-11-19T17:48:42,405 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,409 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T17:48:42,409 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,409 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 42eb2452c7f2a30803f85b3a18651fb5: Waiting for close lock at 1732038522405Running coprocessor pre-close hooks at 1732038522405Disabling compacts and flushes for region at 1732038522405Disabling writes for close at 1732038522405Writing region close event to WAL at 1732038522405Running coprocessor post-close hooks at 1732038522409 (+4 ms)Closed at 1732038522409 2024-11-19T17:48:42,410 WARN [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 42eb2452c7f2a30803f85b3a18651fb5 to self. 2024-11-19T17:48:42,411 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,412 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=42eb2452c7f2a30803f85b3a18651fb5, regionState=CLOSED 2024-11-19T17:48:42,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:48:42,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-19T17:48:42,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 in 163 msec 2024-11-19T17:48:42,417 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, REOPEN/MOVE; state=CLOSED, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=true 2024-11-19T17:48:42,567 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=42eb2452c7f2a30803f85b3a18651fb5, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:42,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, REOPEN/MOVE because future has completed 2024-11-19T17:48:42,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:48:42,724 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,724 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:48:42,724 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,724 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:48:42,724 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,724 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,728 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,735 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42eb2452c7f2a30803f85b3a18651fb5 columnFamilyName A 2024-11-19T17:48:42,737 DEBUG [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:42,737 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(327): Store=42eb2452c7f2a30803f85b3a18651fb5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:42,738 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,739 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42eb2452c7f2a30803f85b3a18651fb5 columnFamilyName B 2024-11-19T17:48:42,739 DEBUG [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:42,739 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(327): Store=42eb2452c7f2a30803f85b3a18651fb5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:42,739 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,740 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42eb2452c7f2a30803f85b3a18651fb5 columnFamilyName C 2024-11-19T17:48:42,740 DEBUG [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:42,740 INFO [StoreOpener-42eb2452c7f2a30803f85b3a18651fb5-1 {}] regionserver.HStore(327): Store=42eb2452c7f2a30803f85b3a18651fb5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:48:42,740 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,741 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,742 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,743 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,743 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,743 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:48:42,745 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,746 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 42eb2452c7f2a30803f85b3a18651fb5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69795355, jitterRate=0.040031835436820984}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:48:42,746 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,747 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 42eb2452c7f2a30803f85b3a18651fb5: Running coprocessor pre-open hook at 1732038522724Writing region info on filesystem at 1732038522724Initializing all the Stores at 1732038522725 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038522725Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038522725Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038522726 (+1 ms)Cleaning up temporary data from old regions at 1732038522743 (+17 ms)Running coprocessor post-open hooks at 1732038522746 (+3 ms)Region opened successfully at 1732038522747 (+1 ms) 2024-11-19T17:48:42,748 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., pid=39, masterSystemTime=1732038522721 2024-11-19T17:48:42,750 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,750 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,751 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=42eb2452c7f2a30803f85b3a18651fb5, regionState=OPEN, openSeqNum=5, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:42,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:48:42,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-19T17:48:42,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 in 184 msec 2024-11-19T17:48:42,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T17:48:42,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, REOPEN/MOVE in 506 msec 2024-11-19T17:48:42,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T17:48:42,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-19T17:48:42,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 944 msec 2024-11-19T17:48:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-19T17:48:42,766 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:42,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f00681c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f321aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,771 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,772 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,775 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ed35e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,776 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@296869d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7084df20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,780 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3991103b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,782 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebd16ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,783 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67afd003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,785 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4179419a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,787 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c316e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,789 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46460dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,790 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40682de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,792 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae94d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,793 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c6f230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:48:42,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:48:42,795 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:48:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:48:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:48:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35051c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:48:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:48:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:48:42,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:48:42,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75874e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:48:42,798 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,799 DEBUG [Thread-692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,799 DEBUG [Thread-690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,799 DEBUG [Thread-701 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,800 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,802 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:42,804 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T17:48:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T17:48:42,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,811 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:42,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:42,814 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,816 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,820 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:48:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:48:42,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:48:42,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:48:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:48:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:42,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:48:42,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ca3d879cacd54061ab294ec7c84e3e2e_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038522812/Put/seqid=0 2024-11-19T17:48:42,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741963_1139 (size=17034) 2024-11-19T17:48:42,874 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:42,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ca3d879cacd54061ab294ec7c84e3e2e_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ca3d879cacd54061ab294ec7c84e3e2e_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:42,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/6322e9245237414fa9aa1966792d2ed8, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:42,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/6322e9245237414fa9aa1966792d2ed8 is 175, key is test_row_0/A:col10/1732038522812/Put/seqid=0 2024-11-19T17:48:42,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:42,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038582908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038582908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T17:48:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:42,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038582910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:42,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038582912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:42,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038582914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:42,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741964_1140 (size=48139) 2024-11-19T17:48:42,961 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/6322e9245237414fa9aa1966792d2ed8 2024-11-19T17:48:42,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T17:48:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:42,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:42,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/d7d1a4afff5c412898bf19e6ce05e132 is 50, key is test_row_0/B:col10/1732038522812/Put/seqid=0 2024-11-19T17:48:43,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741965_1141 (size=12001) 2024-11-19T17:48:43,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038583019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038583029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038583030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038583030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038583039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T17:48:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T17:48:43,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038583229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038583238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038583238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038583239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038583249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T17:48:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/d7d1a4afff5c412898bf19e6ce05e132 2024-11-19T17:48:43,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T17:48:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T17:48:43,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/ad9fcfbc55b94d54bc90ee0d807d61b1 is 50, key is test_row_0/C:col10/1732038522812/Put/seqid=0 2024-11-19T17:48:43,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741966_1142 (size=12001) 2024-11-19T17:48:43,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038583539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038583548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038583548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038583559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038583559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:43,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T17:48:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T17:48:43,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/ad9fcfbc55b94d54bc90ee0d807d61b1 2024-11-19T17:48:43,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/6322e9245237414fa9aa1966792d2ed8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/6322e9245237414fa9aa1966792d2ed8 2024-11-19T17:48:43,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/6322e9245237414fa9aa1966792d2ed8, entries=250, sequenceid=15, filesize=47.0 K 2024-11-19T17:48:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/d7d1a4afff5c412898bf19e6ce05e132 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d7d1a4afff5c412898bf19e6ce05e132 2024-11-19T17:48:43,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d7d1a4afff5c412898bf19e6ce05e132, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T17:48:43,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/ad9fcfbc55b94d54bc90ee0d807d61b1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ad9fcfbc55b94d54bc90ee0d807d61b1 2024-11-19T17:48:43,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ad9fcfbc55b94d54bc90ee0d807d61b1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T17:48:43,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T17:48:43,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:43,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:43,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 42eb2452c7f2a30803f85b3a18651fb5 in 1071ms, sequenceid=15, compaction requested=false 2024-11-19T17:48:43,897 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:48:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:43,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T17:48:44,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T17:48:44,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:44,051 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:48:44,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:44,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197f4af5285e1f43e8aa7603fb188fc6a4_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038522866/Put/seqid=0 2024-11-19T17:48:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038584069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038584075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038584077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038584079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038584079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741967_1143 (size=12154) 2024-11-19T17:48:44,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:44,105 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197f4af5285e1f43e8aa7603fb188fc6a4_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197f4af5285e1f43e8aa7603fb188fc6a4_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:44,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/75db6c31ac0d4b04ab1a8907a40c2c3d, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:44,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/75db6c31ac0d4b04ab1a8907a40c2c3d is 175, key is test_row_0/A:col10/1732038522866/Put/seqid=0 2024-11-19T17:48:44,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741968_1144 (size=30955) 2024-11-19T17:48:44,134 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/75db6c31ac0d4b04ab1a8907a40c2c3d 2024-11-19T17:48:44,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/31063628d54845948755e3ed91f9dcfd is 50, key is test_row_0/B:col10/1732038522866/Put/seqid=0 2024-11-19T17:48:44,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741969_1145 (size=12001) 2024-11-19T17:48:44,186 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/31063628d54845948755e3ed91f9dcfd 2024-11-19T17:48:44,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038584189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038584189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038584189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038584189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/f96a681d27f74ad99eb750f7c2e2f670 is 50, key is test_row_0/C:col10/1732038522866/Put/seqid=0 2024-11-19T17:48:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741970_1146 (size=12001) 2024-11-19T17:48:44,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038584399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038584399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038584399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038584399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,490 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:48:44,606 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/f96a681d27f74ad99eb750f7c2e2f670 2024-11-19T17:48:44,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/75db6c31ac0d4b04ab1a8907a40c2c3d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/75db6c31ac0d4b04ab1a8907a40c2c3d 2024-11-19T17:48:44,618 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/75db6c31ac0d4b04ab1a8907a40c2c3d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-19T17:48:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/31063628d54845948755e3ed91f9dcfd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/31063628d54845948755e3ed91f9dcfd 2024-11-19T17:48:44,627 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/31063628d54845948755e3ed91f9dcfd, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T17:48:44,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/f96a681d27f74ad99eb750f7c2e2f670 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/f96a681d27f74ad99eb750f7c2e2f670 2024-11-19T17:48:44,637 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/f96a681d27f74ad99eb750f7c2e2f670, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T17:48:44,638 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 42eb2452c7f2a30803f85b3a18651fb5 in 587ms, sequenceid=40, compaction requested=false 2024-11-19T17:48:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T17:48:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T17:48:44,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T17:48:44,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8290 sec 2024-11-19T17:48:44,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.8400 sec 2024-11-19T17:48:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:44,711 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:44,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:48:44,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0a06242e5b549baa7061549ff118f28_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038524710/Put/seqid=0 2024-11-19T17:48:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741971_1147 (size=14594) 2024-11-19T17:48:44,734 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:44,742 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0a06242e5b549baa7061549ff118f28_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0a06242e5b549baa7061549ff118f28_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:44,743 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/246f1696bac34bedb63801757ada2513, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:44,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/246f1696bac34bedb63801757ada2513 is 175, key is test_row_0/A:col10/1732038524710/Put/seqid=0 2024-11-19T17:48:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038584768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038584774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038584776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038584780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741972_1148 (size=39549) 2024-11-19T17:48:44,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/246f1696bac34bedb63801757ada2513 2024-11-19T17:48:44,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a28697f1b5dd4f58bbfff538d10d8113 is 50, key is test_row_0/B:col10/1732038524710/Put/seqid=0 2024-11-19T17:48:44,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741973_1149 (size=12001) 2024-11-19T17:48:44,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038584888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038584888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038584888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038584888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T17:48:44,959 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:44,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T17:48:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T17:48:44,962 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:44,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:44,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T17:48:45,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038585088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,095 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038585099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038585099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038585099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038585099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T17:48:45,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:45,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a28697f1b5dd4f58bbfff538d10d8113 2024-11-19T17:48:45,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/e432050e88834c74916d53620111a259 is 50, key is test_row_0/C:col10/1732038524710/Put/seqid=0 2024-11-19T17:48:45,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741974_1150 (size=12001) 2024-11-19T17:48:45,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T17:48:45,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:45,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T17:48:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038585409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038585409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038585409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038585409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T17:48:45,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:45,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T17:48:45,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:45,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T17:48:45,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/e432050e88834c74916d53620111a259 2024-11-19T17:48:45,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/246f1696bac34bedb63801757ada2513 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/246f1696bac34bedb63801757ada2513 2024-11-19T17:48:45,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/246f1696bac34bedb63801757ada2513, entries=200, sequenceid=52, filesize=38.6 K 2024-11-19T17:48:45,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a28697f1b5dd4f58bbfff538d10d8113 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a28697f1b5dd4f58bbfff538d10d8113 2024-11-19T17:48:45,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a28697f1b5dd4f58bbfff538d10d8113, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T17:48:45,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/e432050e88834c74916d53620111a259 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e432050e88834c74916d53620111a259 2024-11-19T17:48:45,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e432050e88834c74916d53620111a259, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T17:48:45,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 42eb2452c7f2a30803f85b3a18651fb5 in 957ms, sequenceid=52, compaction requested=true 2024-11-19T17:48:45,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:45,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:45,669 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:45,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:45,669 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:45,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:45,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:45,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:45,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:45,670 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:45,670 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/A is initiating minor compaction (all files) 2024-11-19T17:48:45,670 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/A in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,670 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/6322e9245237414fa9aa1966792d2ed8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/75db6c31ac0d4b04ab1a8907a40c2c3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/246f1696bac34bedb63801757ada2513] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=115.9 K 2024-11-19T17:48:45,670 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,671 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/6322e9245237414fa9aa1966792d2ed8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/75db6c31ac0d4b04ab1a8907a40c2c3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/246f1696bac34bedb63801757ada2513] 2024-11-19T17:48:45,671 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:45,671 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/B is initiating minor compaction (all files) 2024-11-19T17:48:45,671 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/B in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,671 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d7d1a4afff5c412898bf19e6ce05e132, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/31063628d54845948755e3ed91f9dcfd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a28697f1b5dd4f58bbfff538d10d8113] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=35.2 K 2024-11-19T17:48:45,671 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6322e9245237414fa9aa1966792d2ed8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732038522809 2024-11-19T17:48:45,672 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d7d1a4afff5c412898bf19e6ce05e132, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732038522812 2024-11-19T17:48:45,673 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75db6c31ac0d4b04ab1a8907a40c2c3d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038522866 2024-11-19T17:48:45,673 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 31063628d54845948755e3ed91f9dcfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038522866 2024-11-19T17:48:45,673 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 246f1696bac34bedb63801757ada2513, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732038524071 2024-11-19T17:48:45,673 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a28697f1b5dd4f58bbfff538d10d8113, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732038524071 2024-11-19T17:48:45,682 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:45,684 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119577624d682c346c2b63aeb66f6c4afbd_42eb2452c7f2a30803f85b3a18651fb5 store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:45,685 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#B#compaction#133 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:45,685 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a4bd45351dcd420bb6f945af362dcc20 is 50, key is test_row_0/B:col10/1732038524710/Put/seqid=0 2024-11-19T17:48:45,693 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119577624d682c346c2b63aeb66f6c4afbd_42eb2452c7f2a30803f85b3a18651fb5, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:45,694 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119577624d682c346c2b63aeb66f6c4afbd_42eb2452c7f2a30803f85b3a18651fb5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:45,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741975_1151 (size=12104) 2024-11-19T17:48:45,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741976_1152 (size=4469) 2024-11-19T17:48:45,725 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#A#compaction#132 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:45,726 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5149b4c4efb34de48e56407acb7b3304 is 175, key is test_row_0/A:col10/1732038524710/Put/seqid=0 2024-11-19T17:48:45,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T17:48:45,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,733 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:48:45,735 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a4bd45351dcd420bb6f945af362dcc20 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a4bd45351dcd420bb6f945af362dcc20 2024-11-19T17:48:45,748 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/B of 42eb2452c7f2a30803f85b3a18651fb5 into a4bd45351dcd420bb6f945af362dcc20(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:45,748 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:45,748 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/B, priority=13, startTime=1732038525669; duration=0sec 2024-11-19T17:48:45,748 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:45,748 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:B 2024-11-19T17:48:45,748 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:45,749 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:45,749 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/C is initiating minor compaction (all files) 2024-11-19T17:48:45,749 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/C in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:45,749 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ad9fcfbc55b94d54bc90ee0d807d61b1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/f96a681d27f74ad99eb750f7c2e2f670, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e432050e88834c74916d53620111a259] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=35.2 K 2024-11-19T17:48:45,750 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ad9fcfbc55b94d54bc90ee0d807d61b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732038522812 2024-11-19T17:48:45,750 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting f96a681d27f74ad99eb750f7c2e2f670, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038522866 2024-11-19T17:48:45,751 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting e432050e88834c74916d53620111a259, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732038524071 2024-11-19T17:48:45,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741977_1153 (size=31058) 2024-11-19T17:48:45,768 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5149b4c4efb34de48e56407acb7b3304 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5149b4c4efb34de48e56407acb7b3304 2024-11-19T17:48:45,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111999844ed489f04ca89b5227ae90d65c4c_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038524744/Put/seqid=0 2024-11-19T17:48:45,780 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:45,781 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/7b88587e2558465fba4a241b03937aae is 50, key is test_row_0/C:col10/1732038524710/Put/seqid=0 2024-11-19T17:48:45,783 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/A of 42eb2452c7f2a30803f85b3a18651fb5 into 5149b4c4efb34de48e56407acb7b3304(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:45,783 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:45,783 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/A, priority=13, startTime=1732038525668; duration=0sec 2024-11-19T17:48:45,783 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:45,783 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:A 2024-11-19T17:48:45,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741978_1154 (size=12154) 2024-11-19T17:48:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:45,820 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111999844ed489f04ca89b5227ae90d65c4c_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111999844ed489f04ca89b5227ae90d65c4c_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:45,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/edd57cb4a1814d38a1c28a650c5dfcc6, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/edd57cb4a1814d38a1c28a650c5dfcc6 is 175, key is test_row_0/A:col10/1732038524744/Put/seqid=0 2024-11-19T17:48:45,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741979_1155 (size=12104) 2024-11-19T17:48:45,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741980_1156 (size=30955) 2024-11-19T17:48:45,837 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/edd57cb4a1814d38a1c28a650c5dfcc6 2024-11-19T17:48:45,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/dad94aa1aff147538f6872be7b78406b is 50, key is test_row_0/B:col10/1732038524744/Put/seqid=0 2024-11-19T17:48:45,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741981_1157 (size=12001) 2024-11-19T17:48:45,860 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/dad94aa1aff147538f6872be7b78406b 2024-11-19T17:48:45,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c63c5691fc6743448ef3918b61530729 is 50, key is test_row_0/C:col10/1732038524744/Put/seqid=0 2024-11-19T17:48:45,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741982_1158 (size=12001) 2024-11-19T17:48:45,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:45,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038585931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038585931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038585932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038585933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,030 INFO [master/26d3106abcfe:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T17:48:46,030 INFO [master/26d3106abcfe:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T17:48:46,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038586039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038586039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038586039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038586039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T17:48:46,233 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/7b88587e2558465fba4a241b03937aae as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b88587e2558465fba4a241b03937aae 2024-11-19T17:48:46,238 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/C of 42eb2452c7f2a30803f85b3a18651fb5 into 7b88587e2558465fba4a241b03937aae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:46,238 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:46,238 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/C, priority=13, startTime=1732038525669; duration=0sec 2024-11-19T17:48:46,238 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:46,238 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:C 2024-11-19T17:48:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038586249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038586249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038586251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038586251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,299 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c63c5691fc6743448ef3918b61530729 2024-11-19T17:48:46,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/edd57cb4a1814d38a1c28a650c5dfcc6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/edd57cb4a1814d38a1c28a650c5dfcc6 2024-11-19T17:48:46,311 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/edd57cb4a1814d38a1c28a650c5dfcc6, entries=150, sequenceid=76, filesize=30.2 K 2024-11-19T17:48:46,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/dad94aa1aff147538f6872be7b78406b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dad94aa1aff147538f6872be7b78406b 2024-11-19T17:48:46,323 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dad94aa1aff147538f6872be7b78406b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T17:48:46,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c63c5691fc6743448ef3918b61530729 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c63c5691fc6743448ef3918b61530729 2024-11-19T17:48:46,330 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c63c5691fc6743448ef3918b61530729, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T17:48:46,332 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 42eb2452c7f2a30803f85b3a18651fb5 in 599ms, sequenceid=76, compaction requested=false 2024-11-19T17:48:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T17:48:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T17:48:46,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T17:48:46,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3730 sec 2024-11-19T17:48:46,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.3790 sec 2024-11-19T17:48:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:46,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:48:46,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119efb388da5ccb4f57a78d28e7c1dc3d10_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038526561/Put/seqid=0 2024-11-19T17:48:46,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741983_1159 (size=12154) 2024-11-19T17:48:46,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:46,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,596 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119efb388da5ccb4f57a78d28e7c1dc3d10_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119efb388da5ccb4f57a78d28e7c1dc3d10_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038586590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/c756f554cb3240b3acda3524ba1684bc, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:46,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/c756f554cb3240b3acda3524ba1684bc is 175, key is test_row_0/A:col10/1732038526561/Put/seqid=0 2024-11-19T17:48:46,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038586594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038586596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038586596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741984_1160 (size=30955) 2024-11-19T17:48:46,633 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/c756f554cb3240b3acda3524ba1684bc 2024-11-19T17:48:46,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/88e3e7446c324fd4b598c8494e39bc41 is 50, key is test_row_0/B:col10/1732038526561/Put/seqid=0 2024-11-19T17:48:46,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741985_1161 (size=12001) 2024-11-19T17:48:46,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/88e3e7446c324fd4b598c8494e39bc41 2024-11-19T17:48:46,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/441233117a13433f8ca6931a66c22fed is 50, key is test_row_0/C:col10/1732038526561/Put/seqid=0 2024-11-19T17:48:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741986_1162 (size=12001) 2024-11-19T17:48:46,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/441233117a13433f8ca6931a66c22fed 2024-11-19T17:48:46,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/c756f554cb3240b3acda3524ba1684bc as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c756f554cb3240b3acda3524ba1684bc 2024-11-19T17:48:46,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c756f554cb3240b3acda3524ba1684bc, entries=150, sequenceid=94, filesize=30.2 K 2024-11-19T17:48:46,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/88e3e7446c324fd4b598c8494e39bc41 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/88e3e7446c324fd4b598c8494e39bc41 2024-11-19T17:48:46,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/88e3e7446c324fd4b598c8494e39bc41, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:48:46,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/441233117a13433f8ca6931a66c22fed as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/441233117a13433f8ca6931a66c22fed 2024-11-19T17:48:46,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038586709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038586709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/441233117a13433f8ca6931a66c22fed, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:48:46,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038586720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038586720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 42eb2452c7f2a30803f85b3a18651fb5 in 161ms, sequenceid=94, compaction requested=true 2024-11-19T17:48:46,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:46,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:46,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:46,723 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:46,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:46,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:46,723 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:46,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:46,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:46,724 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:46,724 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/B is initiating minor compaction (all files) 2024-11-19T17:48:46,725 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/B in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:46,725 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a4bd45351dcd420bb6f945af362dcc20, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dad94aa1aff147538f6872be7b78406b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/88e3e7446c324fd4b598c8494e39bc41] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=35.3 K 2024-11-19T17:48:46,725 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:46,725 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/A is initiating minor compaction (all files) 2024-11-19T17:48:46,725 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/A in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:46,725 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5149b4c4efb34de48e56407acb7b3304, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/edd57cb4a1814d38a1c28a650c5dfcc6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c756f554cb3240b3acda3524ba1684bc] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=90.8 K 2024-11-19T17:48:46,725 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:46,725 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5149b4c4efb34de48e56407acb7b3304, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/edd57cb4a1814d38a1c28a650c5dfcc6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c756f554cb3240b3acda3524ba1684bc] 2024-11-19T17:48:46,726 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a4bd45351dcd420bb6f945af362dcc20, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732038524071 2024-11-19T17:48:46,726 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5149b4c4efb34de48e56407acb7b3304, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732038524071 2024-11-19T17:48:46,726 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting dad94aa1aff147538f6872be7b78406b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732038524744 2024-11-19T17:48:46,726 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting edd57cb4a1814d38a1c28a650c5dfcc6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732038524744 2024-11-19T17:48:46,727 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 88e3e7446c324fd4b598c8494e39bc41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038525931 2024-11-19T17:48:46,727 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c756f554cb3240b3acda3524ba1684bc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038525931 2024-11-19T17:48:46,747 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:46,747 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a88ae4f49dea4a508623d2ed7bebd1cc is 50, key is test_row_0/B:col10/1732038526561/Put/seqid=0 2024-11-19T17:48:46,749 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:46,759 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119148d686a4e8242c3912711d05a470c46_42eb2452c7f2a30803f85b3a18651fb5 store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:46,763 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119148d686a4e8242c3912711d05a470c46_42eb2452c7f2a30803f85b3a18651fb5, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:46,764 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119148d686a4e8242c3912711d05a470c46_42eb2452c7f2a30803f85b3a18651fb5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:46,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741987_1163 (size=12207) 2024-11-19T17:48:46,797 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a88ae4f49dea4a508623d2ed7bebd1cc as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a88ae4f49dea4a508623d2ed7bebd1cc 2024-11-19T17:48:46,804 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/B of 42eb2452c7f2a30803f85b3a18651fb5 into a88ae4f49dea4a508623d2ed7bebd1cc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:46,805 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:46,805 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/B, priority=13, startTime=1732038526723; duration=0sec 2024-11-19T17:48:46,805 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:46,806 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:B 2024-11-19T17:48:46,806 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:46,809 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:46,809 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/C is initiating minor compaction (all files) 2024-11-19T17:48:46,809 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/C in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:46,809 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b88587e2558465fba4a241b03937aae, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c63c5691fc6743448ef3918b61530729, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/441233117a13433f8ca6931a66c22fed] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=35.3 K 2024-11-19T17:48:46,810 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b88587e2558465fba4a241b03937aae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732038524071 2024-11-19T17:48:46,812 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c63c5691fc6743448ef3918b61530729, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732038524744 2024-11-19T17:48:46,812 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 441233117a13433f8ca6931a66c22fed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038525931 2024-11-19T17:48:46,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741988_1164 (size=4469) 2024-11-19T17:48:46,827 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#A#compaction#142 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:46,828 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/e46c792fdda045e781baa17a2d26ddb9 is 175, key is test_row_0/A:col10/1732038526561/Put/seqid=0 2024-11-19T17:48:46,841 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#C#compaction#143 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:46,842 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/66ac4a9b49324fba97d8c954c061a31e is 50, key is test_row_0/C:col10/1732038526561/Put/seqid=0 2024-11-19T17:48:46,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741989_1165 (size=31161) 2024-11-19T17:48:46,870 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/e46c792fdda045e781baa17a2d26ddb9 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e46c792fdda045e781baa17a2d26ddb9 2024-11-19T17:48:46,877 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/A of 42eb2452c7f2a30803f85b3a18651fb5 into e46c792fdda045e781baa17a2d26ddb9(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:46,877 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:46,877 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/A, priority=13, startTime=1732038526723; duration=0sec 2024-11-19T17:48:46,877 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:46,877 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:A 2024-11-19T17:48:46,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741990_1166 (size=12207) 2024-11-19T17:48:46,898 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/66ac4a9b49324fba97d8c954c061a31e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/66ac4a9b49324fba97d8c954c061a31e 2024-11-19T17:48:46,906 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/C of 42eb2452c7f2a30803f85b3a18651fb5 into 66ac4a9b49324fba97d8c954c061a31e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:46,906 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:46,906 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/C, priority=13, startTime=1732038526723; duration=0sec 2024-11-19T17:48:46,906 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:46,906 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:C 2024-11-19T17:48:46,921 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:46,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T17:48:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038586937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038586938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038586939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038586941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:46,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:46,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192fe496a6100745a0b9e5651f229c64aa_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038526594/Put/seqid=0 2024-11-19T17:48:46,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741991_1167 (size=14594) 2024-11-19T17:48:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038587049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038587049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038587058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038587058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T17:48:47,109 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:47,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:47,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038587109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,110 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T17:48:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T17:48:47,112 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:47,114 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:47,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T17:48:47,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038587258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038587259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T17:48:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038587269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038587269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:47,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192fe496a6100745a0b9e5651f229c64aa_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fe496a6100745a0b9e5651f229c64aa_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:47,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5da6107e45f44e7c905db30ff6a69ab0, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:47,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5da6107e45f44e7c905db30ff6a69ab0 is 175, key is test_row_0/A:col10/1732038526594/Put/seqid=0 2024-11-19T17:48:47,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741992_1168 (size=39549) 2024-11-19T17:48:47,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T17:48:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T17:48:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038587568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038587568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T17:48:47,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:47,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038587579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038587579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:47,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T17:48:47,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:47,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T17:48:47,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5da6107e45f44e7c905db30ff6a69ab0 2024-11-19T17:48:47,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/137f6f37f2ef46e89f2168762cd5a882 is 50, key is test_row_0/B:col10/1732038526594/Put/seqid=0 2024-11-19T17:48:47,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741993_1169 (size=12001) 2024-11-19T17:48:47,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T17:48:47,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:47,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:48,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T17:48:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:48,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038588078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038588078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:48,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038588088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:48,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038588099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:48,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T17:48:48,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:48,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:48,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/137f6f37f2ef46e89f2168762cd5a882 2024-11-19T17:48:48,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e is 50, key is test_row_0/C:col10/1732038526594/Put/seqid=0 2024-11-19T17:48:48,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741994_1170 (size=12001) 2024-11-19T17:48:48,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e 2024-11-19T17:48:48,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5da6107e45f44e7c905db30ff6a69ab0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5da6107e45f44e7c905db30ff6a69ab0 2024-11-19T17:48:48,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5da6107e45f44e7c905db30ff6a69ab0, entries=200, sequenceid=120, filesize=38.6 K 2024-11-19T17:48:48,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/137f6f37f2ef46e89f2168762cd5a882 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/137f6f37f2ef46e89f2168762cd5a882 2024-11-19T17:48:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T17:48:48,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/137f6f37f2ef46e89f2168762cd5a882, entries=150, sequenceid=120, filesize=11.7 K 2024-11-19T17:48:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e 2024-11-19T17:48:48,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e, entries=150, sequenceid=120, filesize=11.7 K 2024-11-19T17:48:48,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 42eb2452c7f2a30803f85b3a18651fb5 in 1333ms, sequenceid=120, compaction requested=false 2024-11-19T17:48:48,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:48,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T17:48:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:48,340 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:48:48,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195a59353b3f03423caa50c4599d2791bc_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038526939/Put/seqid=0 2024-11-19T17:48:48,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741995_1171 (size=12254) 2024-11-19T17:48:48,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:48,766 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195a59353b3f03423caa50c4599d2791bc_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195a59353b3f03423caa50c4599d2791bc_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:48,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/e561b311b2544b20971505530f97e70a, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:48,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/e561b311b2544b20971505530f97e70a is 175, key is test_row_0/A:col10/1732038526939/Put/seqid=0 2024-11-19T17:48:48,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741996_1172 (size=31055) 2024-11-19T17:48:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:49,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:49,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038589121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038589123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038589130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038589130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,173 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/e561b311b2544b20971505530f97e70a 2024-11-19T17:48:49,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ba90ec56c6ee435693e34d17519b0d97 is 50, key is test_row_0/B:col10/1732038526939/Put/seqid=0 2024-11-19T17:48:49,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741997_1173 (size=12101) 2024-11-19T17:48:49,187 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ba90ec56c6ee435693e34d17519b0d97 2024-11-19T17:48:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/92e487ee793440118a37587a4c0546f2 is 50, key is test_row_0/C:col10/1732038526939/Put/seqid=0 2024-11-19T17:48:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741998_1174 (size=12101) 2024-11-19T17:48:49,209 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/92e487ee793440118a37587a4c0546f2 2024-11-19T17:48:49,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/e561b311b2544b20971505530f97e70a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e561b311b2544b20971505530f97e70a 2024-11-19T17:48:49,221 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e561b311b2544b20971505530f97e70a, entries=150, sequenceid=133, filesize=30.3 K 2024-11-19T17:48:49,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ba90ec56c6ee435693e34d17519b0d97 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba90ec56c6ee435693e34d17519b0d97 2024-11-19T17:48:49,226 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba90ec56c6ee435693e34d17519b0d97, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:48:49,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/92e487ee793440118a37587a4c0546f2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/92e487ee793440118a37587a4c0546f2 2024-11-19T17:48:49,234 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/92e487ee793440118a37587a4c0546f2, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:48:49,235 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 42eb2452c7f2a30803f85b3a18651fb5 in 895ms, sequenceid=133, compaction requested=true 2024-11-19T17:48:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T17:48:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T17:48:49,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T17:48:49,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1230 sec 2024-11-19T17:48:49,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1320 sec 2024-11-19T17:48:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:49,244 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:49,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T17:48:49,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eb942bd032e14dbfaec2d4152446296f_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038529243/Put/seqid=0 2024-11-19T17:48:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038589250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038589252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038589253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038589257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T17:48:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,261 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T17:48:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:48:49,265 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:49,266 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:49,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:49,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741999_1175 (size=17284) 2024-11-19T17:48:49,291 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:49,297 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eb942bd032e14dbfaec2d4152446296f_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eb942bd032e14dbfaec2d4152446296f_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:49,298 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/606e029731924bf990499a122ab13fad, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:49,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/606e029731924bf990499a122ab13fad is 175, key is test_row_0/A:col10/1732038529243/Put/seqid=0 2024-11-19T17:48:49,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742000_1176 (size=48389) 2024-11-19T17:48:49,306 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/606e029731924bf990499a122ab13fad 2024-11-19T17:48:49,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/640752ed8d4a427581964dd216c59bf1 is 50, key is test_row_0/B:col10/1732038529243/Put/seqid=0 2024-11-19T17:48:49,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742001_1177 (size=12151) 2024-11-19T17:48:49,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/640752ed8d4a427581964dd216c59bf1 2024-11-19T17:48:49,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038589359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038589359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038589359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:48:49,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038589371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/d4ffd27020974c3088be25aa1653c233 is 50, key is test_row_0/C:col10/1732038529243/Put/seqid=0 2024-11-19T17:48:49,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742002_1178 (size=12151) 2024-11-19T17:48:49,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/d4ffd27020974c3088be25aa1653c233 2024-11-19T17:48:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/606e029731924bf990499a122ab13fad as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/606e029731924bf990499a122ab13fad 2024-11-19T17:48:49,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/606e029731924bf990499a122ab13fad, entries=250, sequenceid=159, filesize=47.3 K 2024-11-19T17:48:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/640752ed8d4a427581964dd216c59bf1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/640752ed8d4a427581964dd216c59bf1 2024-11-19T17:48:49,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/640752ed8d4a427581964dd216c59bf1, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T17:48:49,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/d4ffd27020974c3088be25aa1653c233 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d4ffd27020974c3088be25aa1653c233 2024-11-19T17:48:49,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d4ffd27020974c3088be25aa1653c233, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T17:48:49,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 42eb2452c7f2a30803f85b3a18651fb5 in 199ms, sequenceid=159, compaction requested=true 2024-11-19T17:48:49,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:49,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:49,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:49,443 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:49,443 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:49,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:49,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:49,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:49,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:49,445 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:49,445 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/A is initiating minor compaction (all files) 2024-11-19T17:48:49,445 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/A in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,446 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e46c792fdda045e781baa17a2d26ddb9, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5da6107e45f44e7c905db30ff6a69ab0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e561b311b2544b20971505530f97e70a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/606e029731924bf990499a122ab13fad] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=146.6 K 2024-11-19T17:48:49,446 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,446 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e46c792fdda045e781baa17a2d26ddb9, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5da6107e45f44e7c905db30ff6a69ab0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e561b311b2544b20971505530f97e70a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/606e029731924bf990499a122ab13fad] 2024-11-19T17:48:49,446 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:49,446 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/B is initiating minor compaction (all files) 2024-11-19T17:48:49,446 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/B in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,446 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a88ae4f49dea4a508623d2ed7bebd1cc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/137f6f37f2ef46e89f2168762cd5a882, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba90ec56c6ee435693e34d17519b0d97, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/640752ed8d4a427581964dd216c59bf1] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=47.3 K 2024-11-19T17:48:49,447 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a88ae4f49dea4a508623d2ed7bebd1cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038525931 2024-11-19T17:48:49,447 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting e46c792fdda045e781baa17a2d26ddb9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038525931 2024-11-19T17:48:49,447 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 137f6f37f2ef46e89f2168762cd5a882, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732038526594 2024-11-19T17:48:49,448 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ba90ec56c6ee435693e34d17519b0d97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038526933 2024-11-19T17:48:49,448 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da6107e45f44e7c905db30ff6a69ab0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732038526586 2024-11-19T17:48:49,448 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 640752ed8d4a427581964dd216c59bf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038529112 2024-11-19T17:48:49,448 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting e561b311b2544b20971505530f97e70a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038526933 2024-11-19T17:48:49,449 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 606e029731924bf990499a122ab13fad, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038529112 2024-11-19T17:48:49,467 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:49,467 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/b976604cea204f57aeac095ed7634ec7 is 50, key is test_row_0/B:col10/1732038529243/Put/seqid=0 2024-11-19T17:48:49,477 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:49,489 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199f4ba0732a044e8aa83d9228c9b2578b_42eb2452c7f2a30803f85b3a18651fb5 store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:49,493 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199f4ba0732a044e8aa83d9228c9b2578b_42eb2452c7f2a30803f85b3a18651fb5, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:49,493 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199f4ba0732a044e8aa83d9228c9b2578b_42eb2452c7f2a30803f85b3a18651fb5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742003_1179 (size=12493) 2024-11-19T17:48:49,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742004_1180 (size=4469) 2024-11-19T17:48:49,525 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#A#compaction#154 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:49,526 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/a01d9051cb574f9487534caf0f774492 is 175, key is test_row_0/A:col10/1732038529243/Put/seqid=0 2024-11-19T17:48:49,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742005_1181 (size=31447) 2024-11-19T17:48:49,553 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/a01d9051cb574f9487534caf0f774492 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/a01d9051cb574f9487534caf0f774492 2024-11-19T17:48:49,559 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/A of 42eb2452c7f2a30803f85b3a18651fb5 into a01d9051cb574f9487534caf0f774492(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:49,560 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:49,560 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/A, priority=12, startTime=1732038529443; duration=0sec 2024-11-19T17:48:49,560 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:49,560 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:A 2024-11-19T17:48:49,560 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:49,562 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:49,562 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/C is initiating minor compaction (all files) 2024-11-19T17:48:49,562 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/C in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,562 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/66ac4a9b49324fba97d8c954c061a31e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/92e487ee793440118a37587a4c0546f2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d4ffd27020974c3088be25aa1653c233] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=47.3 K 2024-11-19T17:48:49,563 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66ac4a9b49324fba97d8c954c061a31e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038525931 2024-11-19T17:48:49,563 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1994e6406b4e8ba8bdfb5a6dfe1b4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732038526594 2024-11-19T17:48:49,564 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92e487ee793440118a37587a4c0546f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038526933 2024-11-19T17:48:49,564 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4ffd27020974c3088be25aa1653c233, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038529112 2024-11-19T17:48:49,572 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:49,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:48:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:49,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:48:49,587 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:49,587 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/895008d86d1a48fbaa61da807eb5fc30 is 50, key is test_row_0/C:col10/1732038529243/Put/seqid=0 2024-11-19T17:48:49,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111998d8b4721267478a912d1bdc74f63de9_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038529570/Put/seqid=0 2024-11-19T17:48:49,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742006_1182 (size=12493) 2024-11-19T17:48:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038589638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038589646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038589647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038589649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742007_1183 (size=14794) 2024-11-19T17:48:49,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:49,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111998d8b4721267478a912d1bdc74f63de9_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111998d8b4721267478a912d1bdc74f63de9_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:49,671 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/d621824055044b00bd46969071822f8c, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:49,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/d621824055044b00bd46969071822f8c is 175, key is test_row_0/A:col10/1732038529570/Put/seqid=0 2024-11-19T17:48:49,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742008_1184 (size=39749) 2024-11-19T17:48:49,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:49,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:49,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038589759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038589759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038589759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038589759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:49,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:49,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:49,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:48:49,912 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/b976604cea204f57aeac095ed7634ec7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/b976604cea204f57aeac095ed7634ec7 2024-11-19T17:48:49,916 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/B of 42eb2452c7f2a30803f85b3a18651fb5 into b976604cea204f57aeac095ed7634ec7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:49,917 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:49,917 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/B, priority=12, startTime=1732038529443; duration=0sec 2024-11-19T17:48:49,917 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:49,917 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:B 2024-11-19T17:48:49,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038589969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038589969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038589969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038589969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:50,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:50,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,047 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/895008d86d1a48fbaa61da807eb5fc30 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/895008d86d1a48fbaa61da807eb5fc30 2024-11-19T17:48:50,053 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/C of 42eb2452c7f2a30803f85b3a18651fb5 into 895008d86d1a48fbaa61da807eb5fc30(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:50,053 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:50,053 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/C, priority=12, startTime=1732038529443; duration=0sec 2024-11-19T17:48:50,053 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:50,053 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:C 2024-11-19T17:48:50,086 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/d621824055044b00bd46969071822f8c 2024-11-19T17:48:50,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/95146989d82c4510befa4ae9c19343b4 is 50, key is test_row_0/B:col10/1732038529570/Put/seqid=0 2024-11-19T17:48:50,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742009_1185 (size=12151) 2024-11-19T17:48:50,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:50,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:50,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038590278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:50,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038590279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038590279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038590279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:50,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:50,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:48:50,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:50,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:50,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/95146989d82c4510befa4ae9c19343b4 2024-11-19T17:48:50,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/8ceb7e2353f14d7e952c0fa5f9ec911c is 50, key is test_row_0/C:col10/1732038529570/Put/seqid=0 2024-11-19T17:48:50,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742010_1186 (size=12151) 2024-11-19T17:48:50,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:50,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038590789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:50,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:50,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:50,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038590798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038590798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038590798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:50,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/8ceb7e2353f14d7e952c0fa5f9ec911c 2024-11-19T17:48:50,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/d621824055044b00bd46969071822f8c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/d621824055044b00bd46969071822f8c 2024-11-19T17:48:50,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/d621824055044b00bd46969071822f8c, entries=200, sequenceid=171, filesize=38.8 K 2024-11-19T17:48:50,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/95146989d82c4510befa4ae9c19343b4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/95146989d82c4510befa4ae9c19343b4 2024-11-19T17:48:50,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/95146989d82c4510befa4ae9c19343b4, entries=150, sequenceid=171, filesize=11.9 K 2024-11-19T17:48:50,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/8ceb7e2353f14d7e952c0fa5f9ec911c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/8ceb7e2353f14d7e952c0fa5f9ec911c 2024-11-19T17:48:50,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/8ceb7e2353f14d7e952c0fa5f9ec911c, entries=150, sequenceid=171, filesize=11.9 K 2024-11-19T17:48:50,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 42eb2452c7f2a30803f85b3a18651fb5 in 1375ms, sequenceid=171, compaction requested=false 2024-11-19T17:48:50,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:50,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:48:50,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:50,954 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:48:50,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a7486023243c42399e2073a52769a44c_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038529645/Put/seqid=0 2024-11-19T17:48:50,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742011_1187 (size=12304) 2024-11-19T17:48:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:51,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038591142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:51,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038591249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:51,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:51,370 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a7486023243c42399e2073a52769a44c_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a7486023243c42399e2073a52769a44c_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:51,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/2591a407be79422890555c96864e0fad, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:51,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/2591a407be79422890555c96864e0fad is 175, key is test_row_0/A:col10/1732038529645/Put/seqid=0 2024-11-19T17:48:51,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742012_1188 (size=31105) 2024-11-19T17:48:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:48:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038591458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:51,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038591768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:51,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:51,777 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/2591a407be79422890555c96864e0fad 2024-11-19T17:48:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/fb2f21ff863b4577a40efa2d6f96be49 is 50, key is test_row_0/B:col10/1732038529645/Put/seqid=0 2024-11-19T17:48:51,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742013_1189 (size=12151) 2024-11-19T17:48:51,796 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/fb2f21ff863b4577a40efa2d6f96be49 2024-11-19T17:48:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c64b17a9817a46c1b344040eaf24f3cc is 50, key is test_row_0/C:col10/1732038529645/Put/seqid=0 2024-11-19T17:48:51,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038591798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:51,805 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038591809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038591809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:51,810 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,811 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:51,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742014_1190 (size=12151) 2024-11-19T17:48:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038591818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:51,820 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c64b17a9817a46c1b344040eaf24f3cc 2024-11-19T17:48:51,821 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:51,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/2591a407be79422890555c96864e0fad as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/2591a407be79422890555c96864e0fad 2024-11-19T17:48:51,834 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/2591a407be79422890555c96864e0fad, entries=150, sequenceid=198, filesize=30.4 K 2024-11-19T17:48:51,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/fb2f21ff863b4577a40efa2d6f96be49 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/fb2f21ff863b4577a40efa2d6f96be49 2024-11-19T17:48:51,841 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/fb2f21ff863b4577a40efa2d6f96be49, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T17:48:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c64b17a9817a46c1b344040eaf24f3cc as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c64b17a9817a46c1b344040eaf24f3cc 2024-11-19T17:48:51,849 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c64b17a9817a46c1b344040eaf24f3cc, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T17:48:51,851 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 42eb2452c7f2a30803f85b3a18651fb5 in 897ms, sequenceid=198, compaction requested=true 2024-11-19T17:48:51,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:51,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:51,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T17:48:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T17:48:51,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T17:48:51,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5880 sec 2024-11-19T17:48:51,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.5960 sec 2024-11-19T17:48:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:52,280 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:52,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:48:52,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190d46869cc063440a8a9a1bdfa5dd114f_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038532279/Put/seqid=0 2024-11-19T17:48:52,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742015_1191 (size=14794) 2024-11-19T17:48:52,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038592367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038592478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:52,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038592688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:52,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:52,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190d46869cc063440a8a9a1bdfa5dd114f_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d46869cc063440a8a9a1bdfa5dd114f_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:52,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/bd3766a6eb2248e2b913d2baa83c350d, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:52,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/bd3766a6eb2248e2b913d2baa83c350d is 175, key is test_row_0/A:col10/1732038532279/Put/seqid=0 2024-11-19T17:48:52,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742016_1192 (size=39749) 2024-11-19T17:48:52,704 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/bd3766a6eb2248e2b913d2baa83c350d 2024-11-19T17:48:52,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a26f1c5909b4457d8a068b3cb014a1b6 is 50, key is test_row_0/B:col10/1732038532279/Put/seqid=0 2024-11-19T17:48:52,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742017_1193 (size=12151) 2024-11-19T17:48:52,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a26f1c5909b4457d8a068b3cb014a1b6 2024-11-19T17:48:52,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/b24f8bc7a409427dbee5ff1b52e4ce00 is 50, key is test_row_0/C:col10/1732038532279/Put/seqid=0 2024-11-19T17:48:52,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742018_1194 (size=12151) 2024-11-19T17:48:53,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038592998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:53,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:53,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/b24f8bc7a409427dbee5ff1b52e4ce00 2024-11-19T17:48:53,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/bd3766a6eb2248e2b913d2baa83c350d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/bd3766a6eb2248e2b913d2baa83c350d 2024-11-19T17:48:53,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/bd3766a6eb2248e2b913d2baa83c350d, entries=200, sequenceid=210, filesize=38.8 K 2024-11-19T17:48:53,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a26f1c5909b4457d8a068b3cb014a1b6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a26f1c5909b4457d8a068b3cb014a1b6 2024-11-19T17:48:53,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a26f1c5909b4457d8a068b3cb014a1b6, entries=150, sequenceid=210, filesize=11.9 K 2024-11-19T17:48:53,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/b24f8bc7a409427dbee5ff1b52e4ce00 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/b24f8bc7a409427dbee5ff1b52e4ce00 2024-11-19T17:48:53,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/b24f8bc7a409427dbee5ff1b52e4ce00, entries=150, sequenceid=210, filesize=11.9 K 2024-11-19T17:48:53,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 42eb2452c7f2a30803f85b3a18651fb5 in 894ms, sequenceid=210, compaction requested=true 2024-11-19T17:48:53,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:53,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:53,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:53,175 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:53,175 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:53,176 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:53,176 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/A is initiating minor compaction (all files) 2024-11-19T17:48:53,176 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:53,176 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/A in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,176 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/B is initiating minor compaction (all files) 2024-11-19T17:48:53,176 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/B in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,176 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/a01d9051cb574f9487534caf0f774492, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/d621824055044b00bd46969071822f8c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/2591a407be79422890555c96864e0fad, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/bd3766a6eb2248e2b913d2baa83c350d] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=138.7 K 2024-11-19T17:48:53,176 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/b976604cea204f57aeac095ed7634ec7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/95146989d82c4510befa4ae9c19343b4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/fb2f21ff863b4577a40efa2d6f96be49, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a26f1c5909b4457d8a068b3cb014a1b6] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=47.8 K 2024-11-19T17:48:53,176 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,176 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/a01d9051cb574f9487534caf0f774492, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/d621824055044b00bd46969071822f8c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/2591a407be79422890555c96864e0fad, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/bd3766a6eb2248e2b913d2baa83c350d] 2024-11-19T17:48:53,177 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b976604cea204f57aeac095ed7634ec7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038529112 2024-11-19T17:48:53,177 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting a01d9051cb574f9487534caf0f774492, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038529112 2024-11-19T17:48:53,177 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 95146989d82c4510befa4ae9c19343b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732038529249 2024-11-19T17:48:53,177 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting d621824055044b00bd46969071822f8c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732038529249 2024-11-19T17:48:53,179 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2591a407be79422890555c96864e0fad, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038529636 2024-11-19T17:48:53,179 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2f21ff863b4577a40efa2d6f96be49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038529636 2024-11-19T17:48:53,179 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd3766a6eb2248e2b913d2baa83c350d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038531129 2024-11-19T17:48:53,179 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a26f1c5909b4457d8a068b3cb014a1b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038531129 2024-11-19T17:48:53,200 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:53,202 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:53,203 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/56b8f24d67ff4120bd8bb1fe3b9ad90a is 50, key is test_row_0/B:col10/1732038532279/Put/seqid=0 2024-11-19T17:48:53,205 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111942e89c0613c347d599a254cd56716a70_42eb2452c7f2a30803f85b3a18651fb5 store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:53,209 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111942e89c0613c347d599a254cd56716a70_42eb2452c7f2a30803f85b3a18651fb5, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:53,209 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111942e89c0613c347d599a254cd56716a70_42eb2452c7f2a30803f85b3a18651fb5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:53,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742020_1196 (size=4469) 2024-11-19T17:48:53,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742019_1195 (size=12629) 2024-11-19T17:48:53,245 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#A#compaction#166 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:53,246 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/25720389e6bb4fd9b9b8aae762b79311 is 175, key is test_row_0/A:col10/1732038532279/Put/seqid=0 2024-11-19T17:48:53,255 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/56b8f24d67ff4120bd8bb1fe3b9ad90a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56b8f24d67ff4120bd8bb1fe3b9ad90a 2024-11-19T17:48:53,263 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/B of 42eb2452c7f2a30803f85b3a18651fb5 into 56b8f24d67ff4120bd8bb1fe3b9ad90a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:53,263 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:53,263 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/B, priority=12, startTime=1732038533175; duration=0sec 2024-11-19T17:48:53,263 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:53,263 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:B 2024-11-19T17:48:53,263 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:53,265 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:53,266 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/C is initiating minor compaction (all files) 2024-11-19T17:48:53,266 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/C in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,266 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/895008d86d1a48fbaa61da807eb5fc30, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/8ceb7e2353f14d7e952c0fa5f9ec911c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c64b17a9817a46c1b344040eaf24f3cc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/b24f8bc7a409427dbee5ff1b52e4ce00] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=47.8 K 2024-11-19T17:48:53,266 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 895008d86d1a48fbaa61da807eb5fc30, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038529112 2024-11-19T17:48:53,267 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ceb7e2353f14d7e952c0fa5f9ec911c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732038529249 2024-11-19T17:48:53,267 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c64b17a9817a46c1b344040eaf24f3cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038529636 2024-11-19T17:48:53,267 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b24f8bc7a409427dbee5ff1b52e4ce00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038531129 2024-11-19T17:48:53,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742021_1197 (size=31583) 2024-11-19T17:48:53,291 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/25720389e6bb4fd9b9b8aae762b79311 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/25720389e6bb4fd9b9b8aae762b79311 2024-11-19T17:48:53,298 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:53,298 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/A of 42eb2452c7f2a30803f85b3a18651fb5 into 25720389e6bb4fd9b9b8aae762b79311(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:53,299 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:53,299 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/A, priority=12, startTime=1732038533174; duration=0sec 2024-11-19T17:48:53,299 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:53,299 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:A 2024-11-19T17:48:53,299 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/81833bdcb772421390a42eb021c06a2b is 50, key is test_row_0/C:col10/1732038532279/Put/seqid=0 2024-11-19T17:48:53,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742022_1198 (size=12629) 2024-11-19T17:48:53,327 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/81833bdcb772421390a42eb021c06a2b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/81833bdcb772421390a42eb021c06a2b 2024-11-19T17:48:53,333 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/C of 42eb2452c7f2a30803f85b3a18651fb5 into 81833bdcb772421390a42eb021c06a2b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:53,333 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:53,333 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/C, priority=12, startTime=1732038533175; duration=0sec 2024-11-19T17:48:53,334 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:53,334 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:C 2024-11-19T17:48:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:48:53,428 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:53,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T17:48:53,432 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:48:53,433 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:53,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:53,512 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:53,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:48:53,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111941f18b30198e4ffe8732f149f9249fef_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038533509/Put/seqid=0 2024-11-19T17:48:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:48:53,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038593546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:53,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742023_1199 (size=14794) 2024-11-19T17:48:53,560 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:53,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111941f18b30198e4ffe8732f149f9249fef_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941f18b30198e4ffe8732f149f9249fef_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:53,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/7ee45a0735d5497297eae38e2a9dd834, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:53,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/7ee45a0735d5497297eae38e2a9dd834 is 175, key is test_row_0/A:col10/1732038533509/Put/seqid=0 2024-11-19T17:48:53,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742024_1200 (size=39749) 2024-11-19T17:48:53,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:48:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038593658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:53,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:48:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:48:53,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038593818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:53,821 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:53,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038593829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:53,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038593829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:53,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038593829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:53,834 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:53,834 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:53,835 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:53,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038593868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:53,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:53,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:53,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:48:53,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:53,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:53,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:53,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/7ee45a0735d5497297eae38e2a9dd834 2024-11-19T17:48:53,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ea6ef955f17646f6a6661e0a1086c317 is 50, key is test_row_0/B:col10/1732038533509/Put/seqid=0 2024-11-19T17:48:54,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742025_1201 (size=12151) 2024-11-19T17:48:54,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ea6ef955f17646f6a6661e0a1086c317 2024-11-19T17:48:54,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/9db754fac5ee4a94874ba5a39b117b1f is 50, key is test_row_0/C:col10/1732038533509/Put/seqid=0 2024-11-19T17:48:54,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742026_1202 (size=12151) 2024-11-19T17:48:54,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:48:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:54,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:48:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038594178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:54,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:48:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:54,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:48:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:54,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:54,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/9db754fac5ee4a94874ba5a39b117b1f 2024-11-19T17:48:54,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/7ee45a0735d5497297eae38e2a9dd834 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/7ee45a0735d5497297eae38e2a9dd834 2024-11-19T17:48:54,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/7ee45a0735d5497297eae38e2a9dd834, entries=200, sequenceid=238, filesize=38.8 K 2024-11-19T17:48:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ea6ef955f17646f6a6661e0a1086c317 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ea6ef955f17646f6a6661e0a1086c317 2024-11-19T17:48:54,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ea6ef955f17646f6a6661e0a1086c317, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T17:48:54,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/9db754fac5ee4a94874ba5a39b117b1f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/9db754fac5ee4a94874ba5a39b117b1f 2024-11-19T17:48:54,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/9db754fac5ee4a94874ba5a39b117b1f, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T17:48:54,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 42eb2452c7f2a30803f85b3a18651fb5 in 946ms, sequenceid=238, compaction requested=false 2024-11-19T17:48:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:54,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:48:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:54,506 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:48:54,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111911d30a6350684cdc9335e366ab9a0be8_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038533541/Put/seqid=0 2024-11-19T17:48:54,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742027_1203 (size=12304) 2024-11-19T17:48:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:48:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:54,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038594765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:54,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038594879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:54,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:54,921 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111911d30a6350684cdc9335e366ab9a0be8_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911d30a6350684cdc9335e366ab9a0be8_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/807bf05c139343fb8f35ced8ce3c15ab, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/807bf05c139343fb8f35ced8ce3c15ab is 175, key is test_row_0/A:col10/1732038533541/Put/seqid=0 2024-11-19T17:48:54,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742028_1204 (size=31105) 2024-11-19T17:48:54,926 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/807bf05c139343fb8f35ced8ce3c15ab 2024-11-19T17:48:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/7a5ff037c15c45bb90c6cb3226f187fc is 50, key is test_row_0/B:col10/1732038533541/Put/seqid=0 2024-11-19T17:48:54,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742029_1205 (size=12151) 2024-11-19T17:48:54,939 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/7a5ff037c15c45bb90c6cb3226f187fc 2024-11-19T17:48:54,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/cf5808343edc4e02aa42bedfda0567b4 is 50, key is test_row_0/C:col10/1732038533541/Put/seqid=0 2024-11-19T17:48:54,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742030_1206 (size=12151) 2024-11-19T17:48:54,958 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/cf5808343edc4e02aa42bedfda0567b4 2024-11-19T17:48:54,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/807bf05c139343fb8f35ced8ce3c15ab as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/807bf05c139343fb8f35ced8ce3c15ab 2024-11-19T17:48:54,968 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/807bf05c139343fb8f35ced8ce3c15ab, entries=150, sequenceid=249, filesize=30.4 K 2024-11-19T17:48:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/7a5ff037c15c45bb90c6cb3226f187fc as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/7a5ff037c15c45bb90c6cb3226f187fc 2024-11-19T17:48:54,977 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/7a5ff037c15c45bb90c6cb3226f187fc, entries=150, sequenceid=249, filesize=11.9 K 2024-11-19T17:48:54,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/cf5808343edc4e02aa42bedfda0567b4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/cf5808343edc4e02aa42bedfda0567b4 2024-11-19T17:48:54,989 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/cf5808343edc4e02aa42bedfda0567b4, entries=150, sequenceid=249, filesize=11.9 K 2024-11-19T17:48:54,990 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 42eb2452c7f2a30803f85b3a18651fb5 in 484ms, sequenceid=249, compaction requested=true 2024-11-19T17:48:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T17:48:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T17:48:54,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T17:48:54,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5590 sec 2024-11-19T17:48:55,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5660 sec 2024-11-19T17:48:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:55,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:55,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T17:48:55,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ad7382647ff44519f49c44ea2127a0d_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038534749/Put/seqid=0 2024-11-19T17:48:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038595111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:55,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742031_1207 (size=14994) 2024-11-19T17:48:55,122 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:55,126 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ad7382647ff44519f49c44ea2127a0d_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ad7382647ff44519f49c44ea2127a0d_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:55,127 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/1208d0264bd743c8ae685e316a7cbf77, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:55,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/1208d0264bd743c8ae685e316a7cbf77 is 175, key is test_row_0/A:col10/1732038534749/Put/seqid=0 2024-11-19T17:48:55,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742032_1208 (size=39949) 2024-11-19T17:48:55,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038595219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:55,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038595428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:55,534 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/1208d0264bd743c8ae685e316a7cbf77 2024-11-19T17:48:55,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/1e69a1c0dbf842fba5f2ee3704d8f3b3 is 50, key is test_row_0/B:col10/1732038534749/Put/seqid=0 2024-11-19T17:48:55,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742033_1209 (size=12301) 2024-11-19T17:48:55,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/1e69a1c0dbf842fba5f2ee3704d8f3b3 2024-11-19T17:48:55,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/dc029795a9f246e99bd94ecc795e8548 is 50, key is test_row_0/C:col10/1732038534749/Put/seqid=0 2024-11-19T17:48:55,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742034_1210 (size=12301) 2024-11-19T17:48:55,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/dc029795a9f246e99bd94ecc795e8548 2024-11-19T17:48:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:48:55,578 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:55,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T17:48:55,582 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:48:55,583 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:55,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/1208d0264bd743c8ae685e316a7cbf77 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/1208d0264bd743c8ae685e316a7cbf77 2024-11-19T17:48:55,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/1208d0264bd743c8ae685e316a7cbf77, entries=200, sequenceid=275, filesize=39.0 K 2024-11-19T17:48:55,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/1e69a1c0dbf842fba5f2ee3704d8f3b3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/1e69a1c0dbf842fba5f2ee3704d8f3b3 2024-11-19T17:48:55,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/1e69a1c0dbf842fba5f2ee3704d8f3b3, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T17:48:55,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/dc029795a9f246e99bd94ecc795e8548 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/dc029795a9f246e99bd94ecc795e8548 2024-11-19T17:48:55,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/dc029795a9f246e99bd94ecc795e8548, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T17:48:55,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 42eb2452c7f2a30803f85b3a18651fb5 in 517ms, sequenceid=275, compaction requested=true 2024-11-19T17:48:55,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:55,607 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:55,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:55,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:55,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:55,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:55,608 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:55,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:55,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:55,609 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:55,609 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/A is initiating minor compaction (all files) 2024-11-19T17:48:55,609 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/A in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:55,609 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/25720389e6bb4fd9b9b8aae762b79311, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/7ee45a0735d5497297eae38e2a9dd834, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/807bf05c139343fb8f35ced8ce3c15ab, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/1208d0264bd743c8ae685e316a7cbf77] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=139.0 K 2024-11-19T17:48:55,609 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:55,609 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/25720389e6bb4fd9b9b8aae762b79311, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/7ee45a0735d5497297eae38e2a9dd834, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/807bf05c139343fb8f35ced8ce3c15ab, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/1208d0264bd743c8ae685e316a7cbf77] 2024-11-19T17:48:55,610 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:55,610 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/B is initiating minor compaction (all files) 2024-11-19T17:48:55,610 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/B in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:55,610 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56b8f24d67ff4120bd8bb1fe3b9ad90a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ea6ef955f17646f6a6661e0a1086c317, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/7a5ff037c15c45bb90c6cb3226f187fc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/1e69a1c0dbf842fba5f2ee3704d8f3b3] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=48.1 K 2024-11-19T17:48:55,610 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25720389e6bb4fd9b9b8aae762b79311, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038531129 2024-11-19T17:48:55,610 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b8f24d67ff4120bd8bb1fe3b9ad90a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038531129 2024-11-19T17:48:55,610 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee45a0735d5497297eae38e2a9dd834, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732038532356 2024-11-19T17:48:55,610 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6ef955f17646f6a6661e0a1086c317, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732038532364 2024-11-19T17:48:55,611 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807bf05c139343fb8f35ced8ce3c15ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732038533535 2024-11-19T17:48:55,611 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5ff037c15c45bb90c6cb3226f187fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732038533535 2024-11-19T17:48:55,611 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1208d0264bd743c8ae685e316a7cbf77, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038534749 2024-11-19T17:48:55,611 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e69a1c0dbf842fba5f2ee3704d8f3b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038534749 2024-11-19T17:48:55,625 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:55,630 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:55,631 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/d5a446376bda4eaf8bca9de42b03b0bf is 50, key is test_row_0/B:col10/1732038534749/Put/seqid=0 2024-11-19T17:48:55,639 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411190ef38e967e6e4c08917ffdd7860d3c72_42eb2452c7f2a30803f85b3a18651fb5 store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:55,642 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411190ef38e967e6e4c08917ffdd7860d3c72_42eb2452c7f2a30803f85b3a18651fb5, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:55,642 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190ef38e967e6e4c08917ffdd7860d3c72_42eb2452c7f2a30803f85b3a18651fb5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:55,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742035_1211 (size=12915) 2024-11-19T17:48:55,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742036_1212 (size=4469) 2024-11-19T17:48:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:48:55,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T17:48:55,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:55,737 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T17:48:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:55,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:55,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d5e8f6714cdf400bb77e2836dc6670ae_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038535097/Put/seqid=0 2024-11-19T17:48:55,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742037_1213 (size=12454) 2024-11-19T17:48:55,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:55,786 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d5e8f6714cdf400bb77e2836dc6670ae_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d5e8f6714cdf400bb77e2836dc6670ae_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:55,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/20ee1f072d5a4ffba922d85cd96e6404, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:55,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/20ee1f072d5a4ffba922d85cd96e6404 is 175, key is test_row_0/A:col10/1732038535097/Put/seqid=0 2024-11-19T17:48:55,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742038_1214 (size=31255) 2024-11-19T17:48:55,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038595876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:48:55,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038595989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:56,055 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#A#compaction#177 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:56,056 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/8e90e04d1b664792a8a79621ae2f7a6e is 175, key is test_row_0/A:col10/1732038534749/Put/seqid=0 2024-11-19T17:48:56,060 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/d5a446376bda4eaf8bca9de42b03b0bf as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d5a446376bda4eaf8bca9de42b03b0bf 2024-11-19T17:48:56,065 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/B of 42eb2452c7f2a30803f85b3a18651fb5 into d5a446376bda4eaf8bca9de42b03b0bf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:56,065 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:56,065 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/B, priority=12, startTime=1732038535608; duration=0sec 2024-11-19T17:48:56,066 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:56,066 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:B 2024-11-19T17:48:56,066 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:48:56,067 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:48:56,067 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/C is initiating minor compaction (all files) 2024-11-19T17:48:56,067 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/C in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:56,067 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/81833bdcb772421390a42eb021c06a2b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/9db754fac5ee4a94874ba5a39b117b1f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/cf5808343edc4e02aa42bedfda0567b4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/dc029795a9f246e99bd94ecc795e8548] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=48.1 K 2024-11-19T17:48:56,067 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 81833bdcb772421390a42eb021c06a2b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038531129 2024-11-19T17:48:56,070 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db754fac5ee4a94874ba5a39b117b1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732038532364 2024-11-19T17:48:56,071 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting cf5808343edc4e02aa42bedfda0567b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732038533535 2024-11-19T17:48:56,071 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting dc029795a9f246e99bd94ecc795e8548, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038534749 2024-11-19T17:48:56,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742039_1215 (size=31869) 2024-11-19T17:48:56,084 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/8e90e04d1b664792a8a79621ae2f7a6e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/8e90e04d1b664792a8a79621ae2f7a6e 2024-11-19T17:48:56,090 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/A of 42eb2452c7f2a30803f85b3a18651fb5 into 8e90e04d1b664792a8a79621ae2f7a6e(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:56,090 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:56,090 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:56,090 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/A, priority=12, startTime=1732038535607; duration=0sec 2024-11-19T17:48:56,090 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:56,090 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:A 2024-11-19T17:48:56,090 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/270c799c85754e809d8a2b8db71efabd is 50, key is test_row_0/C:col10/1732038534749/Put/seqid=0 2024-11-19T17:48:56,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742040_1216 (size=12915) 2024-11-19T17:48:56,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038596198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:48:56,228 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/20ee1f072d5a4ffba922d85cd96e6404 2024-11-19T17:48:56,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/d193e34f29414545ab651b7419351400 is 50, key is test_row_0/B:col10/1732038535097/Put/seqid=0 2024-11-19T17:48:56,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742041_1217 (size=12301) 2024-11-19T17:48:56,505 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/270c799c85754e809d8a2b8db71efabd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/270c799c85754e809d8a2b8db71efabd 2024-11-19T17:48:56,510 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/C of 42eb2452c7f2a30803f85b3a18651fb5 into 270c799c85754e809d8a2b8db71efabd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:56,510 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:56,510 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/C, priority=12, startTime=1732038535608; duration=0sec 2024-11-19T17:48:56,511 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:56,511 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:C 2024-11-19T17:48:56,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038596509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:56,654 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/d193e34f29414545ab651b7419351400 2024-11-19T17:48:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/d1f86817e5f742ceb75d961d1053cfe0 is 50, key is test_row_0/C:col10/1732038535097/Put/seqid=0 2024-11-19T17:48:56,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742042_1218 (size=12301) 2024-11-19T17:48:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:48:57,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038597019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:57,065 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/d1f86817e5f742ceb75d961d1053cfe0 2024-11-19T17:48:57,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/20ee1f072d5a4ffba922d85cd96e6404 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/20ee1f072d5a4ffba922d85cd96e6404 2024-11-19T17:48:57,074 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/20ee1f072d5a4ffba922d85cd96e6404, entries=150, sequenceid=285, filesize=30.5 K 2024-11-19T17:48:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/d193e34f29414545ab651b7419351400 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d193e34f29414545ab651b7419351400 2024-11-19T17:48:57,078 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d193e34f29414545ab651b7419351400, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T17:48:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/d1f86817e5f742ceb75d961d1053cfe0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d1f86817e5f742ceb75d961d1053cfe0 2024-11-19T17:48:57,084 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d1f86817e5f742ceb75d961d1053cfe0, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T17:48:57,085 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 42eb2452c7f2a30803f85b3a18651fb5 in 1348ms, sequenceid=285, compaction requested=false 2024-11-19T17:48:57,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:57,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:57,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T17:48:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T17:48:57,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T17:48:57,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5030 sec 2024-11-19T17:48:57,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5090 sec 2024-11-19T17:48:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:48:57,729 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:57,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T17:48:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:48:57,731 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:57,731 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:57,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:48:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:57,860 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:48:57,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T17:48:57,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a212ee8e8f0f422e90127a073308ad0b_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038537859/Put/seqid=0 2024-11-19T17:48:57,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742043_1219 (size=14994) 2024-11-19T17:48:57,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038597867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50104 deadline: 1732038597868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:57,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50138 deadline: 1732038597868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:57,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038597869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:57,872 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:57,872 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:57,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:57,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038597979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038597979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038598028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:58,031 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 42eb2452c7f2a30803f85b3a18651fb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:48:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:58,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:48:58,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038598189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:58,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038598189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:58,271 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:58,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a212ee8e8f0f422e90127a073308ad0b_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a212ee8e8f0f422e90127a073308ad0b_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:58,276 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5e4f8a4013924c0fbfcd9278e0dd3691, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:58,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5e4f8a4013924c0fbfcd9278e0dd3691 is 175, key is test_row_0/A:col10/1732038537859/Put/seqid=0 2024-11-19T17:48:58,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742044_1220 (size=39949) 2024-11-19T17:48:58,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:58,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:58,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:48:58,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038598498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038598499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:58,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5e4f8a4013924c0fbfcd9278e0dd3691 2024-11-19T17:48:58,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/078b130d2d434d58ae18d4f18e6abac6 is 50, key is test_row_0/B:col10/1732038537859/Put/seqid=0 2024-11-19T17:48:58,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742045_1221 (size=12301) 2024-11-19T17:48:58,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:58,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:48:58,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:58,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:58,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:59,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038599008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:59,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:48:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038599008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:48:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:48:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:48:59,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/078b130d2d434d58ae18d4f18e6abac6 2024-11-19T17:48:59,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:59,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:59,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:48:59,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:59,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:59,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:59,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/2f02ebddc0034531a13d875c4727a45c is 50, key is test_row_0/C:col10/1732038537859/Put/seqid=0 2024-11-19T17:48:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:48:59,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742046_1222 (size=12301) 2024-11-19T17:48:59,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/2f02ebddc0034531a13d875c4727a45c 2024-11-19T17:48:59,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5e4f8a4013924c0fbfcd9278e0dd3691 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5e4f8a4013924c0fbfcd9278e0dd3691 2024-11-19T17:48:59,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5e4f8a4013924c0fbfcd9278e0dd3691, entries=200, sequenceid=315, filesize=39.0 K 2024-11-19T17:48:59,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/078b130d2d434d58ae18d4f18e6abac6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/078b130d2d434d58ae18d4f18e6abac6 2024-11-19T17:48:59,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/078b130d2d434d58ae18d4f18e6abac6, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T17:48:59,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/2f02ebddc0034531a13d875c4727a45c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2f02ebddc0034531a13d875c4727a45c 2024-11-19T17:48:59,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2f02ebddc0034531a13d875c4727a45c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T17:48:59,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 42eb2452c7f2a30803f85b3a18651fb5 in 1282ms, sequenceid=315, compaction requested=true 2024-11-19T17:48:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:48:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:59,142 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:48:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:59,142 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:48:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:59,146 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:59,146 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:59,146 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/A is initiating minor compaction (all files) 2024-11-19T17:48:59,146 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/B is initiating minor compaction (all files) 2024-11-19T17:48:59,146 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/B in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:59,146 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d5a446376bda4eaf8bca9de42b03b0bf, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d193e34f29414545ab651b7419351400, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/078b130d2d434d58ae18d4f18e6abac6] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=36.6 K 2024-11-19T17:48:59,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/A in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:59,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/8e90e04d1b664792a8a79621ae2f7a6e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/20ee1f072d5a4ffba922d85cd96e6404, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5e4f8a4013924c0fbfcd9278e0dd3691] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=100.7 K 2024-11-19T17:48:59,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:59,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/8e90e04d1b664792a8a79621ae2f7a6e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/20ee1f072d5a4ffba922d85cd96e6404, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5e4f8a4013924c0fbfcd9278e0dd3691] 2024-11-19T17:48:59,147 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a446376bda4eaf8bca9de42b03b0bf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038534749 2024-11-19T17:48:59,147 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d193e34f29414545ab651b7419351400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732038535097 2024-11-19T17:48:59,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e90e04d1b664792a8a79621ae2f7a6e, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038534749 2024-11-19T17:48:59,148 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ee1f072d5a4ffba922d85cd96e6404, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732038535097 2024-11-19T17:48:59,148 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 078b130d2d434d58ae18d4f18e6abac6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732038535871 2024-11-19T17:48:59,148 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e4f8a4013924c0fbfcd9278e0dd3691, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732038535865 2024-11-19T17:48:59,159 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:59,160 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/c76e288d22ab4b1b92fa21b6225ab616 is 50, key is test_row_0/B:col10/1732038537859/Put/seqid=0 2024-11-19T17:48:59,172 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:59,179 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111931fd337e3eba430bb74f50ce4c6fca31_42eb2452c7f2a30803f85b3a18651fb5 store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:59,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742047_1223 (size=13017) 2024-11-19T17:48:59,182 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111931fd337e3eba430bb74f50ce4c6fca31_42eb2452c7f2a30803f85b3a18651fb5, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:59,182 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111931fd337e3eba430bb74f50ce4c6fca31_42eb2452c7f2a30803f85b3a18651fb5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:59,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742048_1224 (size=4469) 2024-11-19T17:48:59,192 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#A#compaction#187 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:59,192 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/c5b72449b94b40bf94d75bf737f289a0 is 175, key is test_row_0/A:col10/1732038537859/Put/seqid=0 2024-11-19T17:48:59,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742049_1225 (size=31971) 2024-11-19T17:48:59,205 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/c5b72449b94b40bf94d75bf737f289a0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c5b72449b94b40bf94d75bf737f289a0 2024-11-19T17:48:59,212 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/A of 42eb2452c7f2a30803f85b3a18651fb5 into c5b72449b94b40bf94d75bf737f289a0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:59,212 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:59,213 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/A, priority=13, startTime=1732038539142; duration=0sec 2024-11-19T17:48:59,213 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:48:59,213 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:A 2024-11-19T17:48:59,213 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:48:59,214 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:48:59,215 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/C is initiating minor compaction (all files) 2024-11-19T17:48:59,215 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/C in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:59,215 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/270c799c85754e809d8a2b8db71efabd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d1f86817e5f742ceb75d961d1053cfe0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2f02ebddc0034531a13d875c4727a45c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=36.6 K 2024-11-19T17:48:59,215 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 270c799c85754e809d8a2b8db71efabd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038534749 2024-11-19T17:48:59,215 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1f86817e5f742ceb75d961d1053cfe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732038535097 2024-11-19T17:48:59,216 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f02ebddc0034531a13d875c4727a45c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732038535871 2024-11-19T17:48:59,225 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:48:59,225 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/a4d4c892df85462ea92a38fa9076d08f is 50, key is test_row_0/C:col10/1732038537859/Put/seqid=0 2024-11-19T17:48:59,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742050_1226 (size=13017) 2024-11-19T17:48:59,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:48:59,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:59,258 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-19T17:48:59,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119123d86b4690146749c356642531eb8a3_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038537862/Put/seqid=0 2024-11-19T17:48:59,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742051_1227 (size=12454) 2024-11-19T17:48:59,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,290 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119123d86b4690146749c356642531eb8a3_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119123d86b4690146749c356642531eb8a3_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:48:59,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5be5e45e4a53429795755812fbf82204, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:48:59,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5be5e45e4a53429795755812fbf82204 is 175, key is test_row_0/A:col10/1732038537862/Put/seqid=0 2024-11-19T17:48:59,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742052_1228 (size=31255) 2024-11-19T17:48:59,297 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5be5e45e4a53429795755812fbf82204 2024-11-19T17:48:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/dcff1415d7d74c318bba3fb48af80f91 is 50, key is test_row_0/B:col10/1732038537862/Put/seqid=0 2024-11-19T17:48:59,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742053_1229 (size=12301) 2024-11-19T17:48:59,323 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/dcff1415d7d74c318bba3fb48af80f91 2024-11-19T17:48:59,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/e56c06922f0e4114ba2f0d05c1ce9050 is 50, key is test_row_0/C:col10/1732038537862/Put/seqid=0 2024-11-19T17:48:59,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742054_1230 (size=12301) 2024-11-19T17:48:59,348 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/e56c06922f0e4114ba2f0d05c1ce9050 2024-11-19T17:48:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5be5e45e4a53429795755812fbf82204 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5be5e45e4a53429795755812fbf82204 2024-11-19T17:48:59,366 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5be5e45e4a53429795755812fbf82204, entries=150, sequenceid=325, filesize=30.5 K 2024-11-19T17:48:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/dcff1415d7d74c318bba3fb48af80f91 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dcff1415d7d74c318bba3fb48af80f91 2024-11-19T17:48:59,374 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dcff1415d7d74c318bba3fb48af80f91, entries=150, sequenceid=325, filesize=12.0 K 2024-11-19T17:48:59,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/e56c06922f0e4114ba2f0d05c1ce9050 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e56c06922f0e4114ba2f0d05c1ce9050 2024-11-19T17:48:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,380 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e56c06922f0e4114ba2f0d05c1ce9050, entries=150, sequenceid=325, filesize=12.0 K 2024-11-19T17:48:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,381 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 42eb2452c7f2a30803f85b3a18651fb5 in 123ms, sequenceid=325, compaction requested=false 2024-11-19T17:48:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:48:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T17:48:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T17:48:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T17:48:59,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6520 sec 2024-11-19T17:48:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6570 sec 2024-11-19T17:48:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,587 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/c76e288d22ab4b1b92fa21b6225ab616 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/c76e288d22ab4b1b92fa21b6225ab616 2024-11-19T17:48:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,595 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/B of 42eb2452c7f2a30803f85b3a18651fb5 into c76e288d22ab4b1b92fa21b6225ab616(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:59,595 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:59,595 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/B, priority=13, startTime=1732038539142; duration=0sec 2024-11-19T17:48:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,595 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:59,595 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:B 2024-11-19T17:48:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,643 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/a4d4c892df85462ea92a38fa9076d08f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/a4d4c892df85462ea92a38fa9076d08f 2024-11-19T17:48:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,649 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/C of 42eb2452c7f2a30803f85b3a18651fb5 into a4d4c892df85462ea92a38fa9076d08f(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:48:59,649 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:48:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,649 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/C, priority=13, startTime=1732038539142; duration=0sec 2024-11-19T17:48:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,649 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:48:59,649 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:C 2024-11-19T17:48:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:48:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:48:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,881 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:48:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-19T17:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T17:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,884 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,885 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:48:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T17:48:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:48:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T17:49:00,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:00,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-19T17:49:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-19T17:49:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T17:49:00,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-11-19T17:49:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 163 msec 2024-11-19T17:49:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:00,105 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:00,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:49:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ad123d6925244540ac6e97c61afc73b9_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038540092/Put/seqid=0 2024-11-19T17:49:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742055_1231 (size=25158) 2024-11-19T17:49:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,133 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ad123d6925244540ac6e97c61afc73b9_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad123d6925244540ac6e97c61afc73b9_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5696ec70eed34000bc38d167f07c98aa, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5696ec70eed34000bc38d167f07c98aa is 175, key is test_row_0/A:col10/1732038540092/Put/seqid=0 2024-11-19T17:49:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742056_1232 (size=74795) 2024-11-19T17:49:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:00,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038600167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038600168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038600174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T17:49:00,199 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:00,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-19T17:49:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:49:00,203 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:00,204 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:00,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038600279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038600279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038600288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:49:00,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:49:00,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:49:00,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038600489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038600489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038600498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:49:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:49:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:49:00,556 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5696ec70eed34000bc38d167f07c98aa 2024-11-19T17:49:00,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/56a7430a65ab4570b44e0add8608886a is 50, key is test_row_0/B:col10/1732038540092/Put/seqid=0 2024-11-19T17:49:00,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742057_1233 (size=12301) 2024-11-19T17:49:00,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:49:00,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:49:00,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038600799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038600799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038600809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:00,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:49:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:49:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:49:00,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:49:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:49:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:00,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:00,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/56a7430a65ab4570b44e0add8608886a 2024-11-19T17:49:00,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c6f97cf346724c1e9fdff8060a0f77c5 is 50, key is test_row_0/C:col10/1732038540092/Put/seqid=0 2024-11-19T17:49:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742058_1234 (size=12301) 2024-11-19T17:49:00,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c6f97cf346724c1e9fdff8060a0f77c5 2024-11-19T17:49:00,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5696ec70eed34000bc38d167f07c98aa as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5696ec70eed34000bc38d167f07c98aa 2024-11-19T17:49:00,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5696ec70eed34000bc38d167f07c98aa, entries=400, sequenceid=339, filesize=73.0 K 2024-11-19T17:49:00,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/56a7430a65ab4570b44e0add8608886a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56a7430a65ab4570b44e0add8608886a 2024-11-19T17:49:00,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56a7430a65ab4570b44e0add8608886a, entries=150, sequenceid=339, filesize=12.0 K 2024-11-19T17:49:00,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/c6f97cf346724c1e9fdff8060a0f77c5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c6f97cf346724c1e9fdff8060a0f77c5 2024-11-19T17:49:01,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c6f97cf346724c1e9fdff8060a0f77c5, entries=150, sequenceid=339, filesize=12.0 K 2024-11-19T17:49:01,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 42eb2452c7f2a30803f85b3a18651fb5 in 898ms, sequenceid=339, compaction requested=true 2024-11-19T17:49:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:01,004 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:49:01,004 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:01,005 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138021 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:01,005 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:01,005 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/B is initiating minor compaction (all files) 2024-11-19T17:49:01,005 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/A is initiating minor compaction (all files) 2024-11-19T17:49:01,005 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/B in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:01,005 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/A in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:01,005 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/c76e288d22ab4b1b92fa21b6225ab616, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dcff1415d7d74c318bba3fb48af80f91, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56a7430a65ab4570b44e0add8608886a] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=36.7 K 2024-11-19T17:49:01,005 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c5b72449b94b40bf94d75bf737f289a0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5be5e45e4a53429795755812fbf82204, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5696ec70eed34000bc38d167f07c98aa] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=134.8 K 2024-11-19T17:49:01,005 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:01,005 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c5b72449b94b40bf94d75bf737f289a0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5be5e45e4a53429795755812fbf82204, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5696ec70eed34000bc38d167f07c98aa] 2024-11-19T17:49:01,006 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c76e288d22ab4b1b92fa21b6225ab616, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732038535871 2024-11-19T17:49:01,006 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5b72449b94b40bf94d75bf737f289a0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732038535871 2024-11-19T17:49:01,006 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting dcff1415d7d74c318bba3fb48af80f91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732038537862 2024-11-19T17:49:01,006 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5be5e45e4a53429795755812fbf82204, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732038537862 2024-11-19T17:49:01,006 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a7430a65ab4570b44e0add8608886a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732038540092 2024-11-19T17:49:01,006 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5696ec70eed34000bc38d167f07c98aa, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732038540063 2024-11-19T17:49:01,021 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:01,023 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#B#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:01,023 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/4c9f3b4b9ee8410591bfc8a58370d886 is 50, key is test_row_0/B:col10/1732038540092/Put/seqid=0 2024-11-19T17:49:01,025 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191c5cc7e119d24798b84f3339e0cce6fa_42eb2452c7f2a30803f85b3a18651fb5 store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:01,028 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191c5cc7e119d24798b84f3339e0cce6fa_42eb2452c7f2a30803f85b3a18651fb5, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:01,029 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c5cc7e119d24798b84f3339e0cce6fa_42eb2452c7f2a30803f85b3a18651fb5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:01,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742059_1235 (size=13119) 2024-11-19T17:49:01,040 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/4c9f3b4b9ee8410591bfc8a58370d886 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/4c9f3b4b9ee8410591bfc8a58370d886 2024-11-19T17:49:01,044 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/B of 42eb2452c7f2a30803f85b3a18651fb5 into 4c9f3b4b9ee8410591bfc8a58370d886(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:01,044 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:01,044 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/B, priority=13, startTime=1732038541004; duration=0sec 2024-11-19T17:49:01,044 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:01,045 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:B 2024-11-19T17:49:01,045 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:01,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742060_1236 (size=4469) 2024-11-19T17:49:01,048 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:01,048 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/C is initiating minor compaction (all files) 2024-11-19T17:49:01,048 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/C in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:01,048 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/a4d4c892df85462ea92a38fa9076d08f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e56c06922f0e4114ba2f0d05c1ce9050, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c6f97cf346724c1e9fdff8060a0f77c5] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=36.7 K 2024-11-19T17:49:01,048 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d4c892df85462ea92a38fa9076d08f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732038535871 2024-11-19T17:49:01,049 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting e56c06922f0e4114ba2f0d05c1ce9050, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732038537862 2024-11-19T17:49:01,049 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f97cf346724c1e9fdff8060a0f77c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732038540092 2024-11-19T17:49:01,058 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:01,059 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/648aa403860643218017da3a34ad71c0 is 50, key is test_row_0/C:col10/1732038540092/Put/seqid=0 2024-11-19T17:49:01,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742061_1237 (size=13119) 2024-11-19T17:49:01,067 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/648aa403860643218017da3a34ad71c0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/648aa403860643218017da3a34ad71c0 2024-11-19T17:49:01,072 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/C of 42eb2452c7f2a30803f85b3a18651fb5 into 648aa403860643218017da3a34ad71c0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:01,072 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:01,072 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/C, priority=13, startTime=1732038541004; duration=0sec 2024-11-19T17:49:01,073 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:01,073 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:C 2024-11-19T17:49:01,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:49:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:01,121 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:49:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b8419b5d26804e7fae979ec6350a2b2d_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038540171/Put/seqid=0 2024-11-19T17:49:01,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742062_1238 (size=12454) 2024-11-19T17:49:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:49:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038601324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038601325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038601326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:49:01,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038601429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038601439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038601439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,447 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#A#compaction#195 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:01,448 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/90c31f3275de48f6aee208e1aeb73e86 is 175, key is test_row_0/A:col10/1732038540092/Put/seqid=0 2024-11-19T17:49:01,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742063_1239 (size=32073) 2024-11-19T17:49:01,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:01,547 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b8419b5d26804e7fae979ec6350a2b2d_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8419b5d26804e7fae979ec6350a2b2d_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:01,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5df63bf5a66d425b8d3c471b944a08f5, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:01,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5df63bf5a66d425b8d3c471b944a08f5 is 175, key is test_row_0/A:col10/1732038540171/Put/seqid=0 2024-11-19T17:49:01,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742064_1240 (size=31255) 2024-11-19T17:49:01,554 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5df63bf5a66d425b8d3c471b944a08f5 2024-11-19T17:49:01,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/8dcf1730dc68405db84c3bff4f438a46 is 50, key is test_row_0/B:col10/1732038540171/Put/seqid=0 2024-11-19T17:49:01,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742065_1241 (size=12301) 2024-11-19T17:49:01,588 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/8dcf1730dc68405db84c3bff4f438a46 2024-11-19T17:49:01,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/3f636fb7dd1348cb9d8a8d2abf330468 is 50, key is test_row_0/C:col10/1732038540171/Put/seqid=0 2024-11-19T17:49:01,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742066_1242 (size=12301) 2024-11-19T17:49:01,606 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/3f636fb7dd1348cb9d8a8d2abf330468 2024-11-19T17:49:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/5df63bf5a66d425b8d3c471b944a08f5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5df63bf5a66d425b8d3c471b944a08f5 2024-11-19T17:49:01,619 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5df63bf5a66d425b8d3c471b944a08f5, entries=150, sequenceid=366, filesize=30.5 K 2024-11-19T17:49:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/8dcf1730dc68405db84c3bff4f438a46 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/8dcf1730dc68405db84c3bff4f438a46 2024-11-19T17:49:01,626 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/8dcf1730dc68405db84c3bff4f438a46, entries=150, sequenceid=366, filesize=12.0 K 2024-11-19T17:49:01,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/3f636fb7dd1348cb9d8a8d2abf330468 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3f636fb7dd1348cb9d8a8d2abf330468 2024-11-19T17:49:01,631 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3f636fb7dd1348cb9d8a8d2abf330468, entries=150, sequenceid=366, filesize=12.0 K 2024-11-19T17:49:01,632 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 42eb2452c7f2a30803f85b3a18651fb5 in 510ms, sequenceid=366, compaction requested=false 2024-11-19T17:49:01,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:01,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:01,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-19T17:49:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-19T17:49:01,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T17:49:01,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-19T17:49:01,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4350 sec 2024-11-19T17:49:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:01,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:49:01,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dee19745a7554a9491020efde6c46545_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038541322/Put/seqid=0 2024-11-19T17:49:01,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742067_1243 (size=14994) 2024-11-19T17:49:01,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038601716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038601717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038601718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038601828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038601828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038601828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:01,856 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/90c31f3275de48f6aee208e1aeb73e86 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/90c31f3275de48f6aee208e1aeb73e86 2024-11-19T17:49:01,862 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/A of 42eb2452c7f2a30803f85b3a18651fb5 into 90c31f3275de48f6aee208e1aeb73e86(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:01,862 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:01,862 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/A, priority=13, startTime=1732038541004; duration=0sec 2024-11-19T17:49:01,862 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:01,862 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:A 2024-11-19T17:49:02,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038602039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:02,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038602039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038602039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:02,068 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:02,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dee19745a7554a9491020efde6c46545_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dee19745a7554a9491020efde6c46545_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:02,072 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/95458fa3d2024ea7a344bea0a8c7d3d0, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:02,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/95458fa3d2024ea7a344bea0a8c7d3d0 is 175, key is test_row_0/A:col10/1732038541322/Put/seqid=0 2024-11-19T17:49:02,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742068_1244 (size=39949) 2024-11-19T17:49:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:49:02,349 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:02,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:02,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50106 deadline: 1732038602349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:02,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50174 deadline: 1732038602349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-19T17:49:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:02,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50110 deadline: 1732038602349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:49:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:02,351 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:02,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:02,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:49:02,478 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/95458fa3d2024ea7a344bea0a8c7d3d0 2024-11-19T17:49:02,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/45bc4684c8554a798d7d695e4c259dd3 is 50, key is test_row_0/B:col10/1732038541322/Put/seqid=0 2024-11-19T17:49:02,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742069_1245 (size=12301) 2024-11-19T17:49:02,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/45bc4684c8554a798d7d695e4c259dd3 2024-11-19T17:49:02,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/61624145cf984408b53f6efe9f47a770 is 50, key is test_row_0/C:col10/1732038541322/Put/seqid=0 2024-11-19T17:49:02,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742070_1246 (size=12301) 2024-11-19T17:49:02,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/61624145cf984408b53f6efe9f47a770 2024-11-19T17:49:02,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T17:49:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:49:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:02,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:02,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/95458fa3d2024ea7a344bea0a8c7d3d0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/95458fa3d2024ea7a344bea0a8c7d3d0 2024-11-19T17:49:02,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/95458fa3d2024ea7a344bea0a8c7d3d0, entries=200, sequenceid=378, filesize=39.0 K 2024-11-19T17:49:02,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/45bc4684c8554a798d7d695e4c259dd3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/45bc4684c8554a798d7d695e4c259dd3 2024-11-19T17:49:02,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/45bc4684c8554a798d7d695e4c259dd3, entries=150, sequenceid=378, filesize=12.0 K 2024-11-19T17:49:02,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/61624145cf984408b53f6efe9f47a770 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/61624145cf984408b53f6efe9f47a770 2024-11-19T17:49:02,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/61624145cf984408b53f6efe9f47a770, entries=150, sequenceid=378, filesize=12.0 K 2024-11-19T17:49:02,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 42eb2452c7f2a30803f85b3a18651fb5 in 880ms, sequenceid=378, compaction requested=true 2024-11-19T17:49:02,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:02,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:02,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:02,522 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:02,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:02,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:02,523 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:02,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42eb2452c7f2a30803f85b3a18651fb5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:02,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:02,524 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:02,524 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/A is initiating minor compaction (all files) 2024-11-19T17:49:02,524 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/A in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:02,524 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/90c31f3275de48f6aee208e1aeb73e86, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5df63bf5a66d425b8d3c471b944a08f5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/95458fa3d2024ea7a344bea0a8c7d3d0] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=100.9 K 2024-11-19T17:49:02,524 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:02,524 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/90c31f3275de48f6aee208e1aeb73e86, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5df63bf5a66d425b8d3c471b944a08f5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/95458fa3d2024ea7a344bea0a8c7d3d0] 2024-11-19T17:49:02,524 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:02,524 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/B is initiating minor compaction (all files) 2024-11-19T17:49:02,524 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/B in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:02,524 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/4c9f3b4b9ee8410591bfc8a58370d886, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/8dcf1730dc68405db84c3bff4f438a46, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/45bc4684c8554a798d7d695e4c259dd3] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=36.8 K 2024-11-19T17:49:02,525 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90c31f3275de48f6aee208e1aeb73e86, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732038540092 2024-11-19T17:49:02,525 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9f3b4b9ee8410591bfc8a58370d886, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732038540092 2024-11-19T17:49:02,525 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5df63bf5a66d425b8d3c471b944a08f5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732038540165 2024-11-19T17:49:02,525 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dcf1730dc68405db84c3bff4f438a46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732038540165 2024-11-19T17:49:02,525 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 45bc4684c8554a798d7d695e4c259dd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732038541322 2024-11-19T17:49:02,525 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95458fa3d2024ea7a344bea0a8c7d3d0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732038541311 2024-11-19T17:49:02,532 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:02,532 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:02,533 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a9dcb412c1634276a0a77b04940ddacc is 50, key is test_row_0/B:col10/1732038541322/Put/seqid=0 2024-11-19T17:49:02,545 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198ad78c74aa33465086367acc09ff7b7f_42eb2452c7f2a30803f85b3a18651fb5 store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:02,547 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198ad78c74aa33465086367acc09ff7b7f_42eb2452c7f2a30803f85b3a18651fb5, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:02,547 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198ad78c74aa33465086367acc09ff7b7f_42eb2452c7f2a30803f85b3a18651fb5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:02,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742071_1247 (size=13221) 2024-11-19T17:49:02,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742072_1248 (size=4469) 2024-11-19T17:49:02,577 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#A#compaction#205 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:02,578 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/f8cc593b530b42b8b2573b945b405793 is 175, key is test_row_0/A:col10/1732038541322/Put/seqid=0 2024-11-19T17:49:02,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742073_1249 (size=32175) 2024-11-19T17:49:02,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T17:49:02,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:02,657 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:49:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:49:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119367948eead3343f981c20525befab04f_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038541715/Put/seqid=0 2024-11-19T17:49:02,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742074_1250 (size=12454) 2024-11-19T17:49:02,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:02,711 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119367948eead3343f981c20525befab04f_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119367948eead3343f981c20525befab04f_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/671b301a39c94f1c820b6ff747791d83, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/671b301a39c94f1c820b6ff747791d83 is 175, key is test_row_0/A:col10/1732038541715/Put/seqid=0 2024-11-19T17:49:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742075_1251 (size=31255) 2024-11-19T17:49:02,730 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/671b301a39c94f1c820b6ff747791d83 2024-11-19T17:49:02,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ef599094dbaa4d8785327803c9694b60 is 50, key is test_row_0/B:col10/1732038541715/Put/seqid=0 2024-11-19T17:49:02,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742076_1252 (size=12301) 2024-11-19T17:49:02,760 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ef599094dbaa4d8785327803c9694b60 2024-11-19T17:49:02,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/3bdbceaefe9b4529a6936a3221eb588e is 50, key is test_row_0/C:col10/1732038541715/Put/seqid=0 2024-11-19T17:49:02,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742077_1253 (size=12301) 2024-11-19T17:49:02,816 INFO [Thread-701 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-701. 2024-11-19T17:49:02,816 DEBUG [Thread-701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:02,816 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,816 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:02,816 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-19T17:49:02,816 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:02,816 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,817 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:02,818 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-11-19T17:49:02,818 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:02,818 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,818 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:02,823 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-11-19T17:49:02,823 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:02,823 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,823 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:02,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. as already flushing 2024-11-19T17:49:02,860 INFO [Thread-692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-692. 2024-11-19T17:49:02,860 INFO [Thread-690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-690. 2024-11-19T17:49:02,860 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-19T17:49:02,860 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:02,860 DEBUG [Thread-690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:02,860 DEBUG [Thread-692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:02,860 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,860 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,860 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,860 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,860 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,860 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:02,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:02,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:02,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:02,959 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/a9dcb412c1634276a0a77b04940ddacc as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a9dcb412c1634276a0a77b04940ddacc 2024-11-19T17:49:02,964 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/B of 42eb2452c7f2a30803f85b3a18651fb5 into a9dcb412c1634276a0a77b04940ddacc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:02,964 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:02,964 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/B, priority=13, startTime=1732038542523; duration=0sec 2024-11-19T17:49:02,964 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:02,964 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:B 2024-11-19T17:49:02,964 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:02,965 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:02,965 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 42eb2452c7f2a30803f85b3a18651fb5/C is initiating minor compaction (all files) 2024-11-19T17:49:02,966 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42eb2452c7f2a30803f85b3a18651fb5/C in TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:02,966 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/648aa403860643218017da3a34ad71c0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3f636fb7dd1348cb9d8a8d2abf330468, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/61624145cf984408b53f6efe9f47a770] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp, totalSize=36.8 K 2024-11-19T17:49:02,966 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 648aa403860643218017da3a34ad71c0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732038540092 2024-11-19T17:49:02,966 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f636fb7dd1348cb9d8a8d2abf330468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732038540165 2024-11-19T17:49:02,967 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 61624145cf984408b53f6efe9f47a770, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732038541322 2024-11-19T17:49:02,973 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42eb2452c7f2a30803f85b3a18651fb5#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:02,974 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/2dc37926436547cf99c252a33f2cfdae is 50, key is test_row_0/C:col10/1732038541322/Put/seqid=0 2024-11-19T17:49:02,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742078_1254 (size=13221) 2024-11-19T17:49:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:49:02,990 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/f8cc593b530b42b8b2573b945b405793 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/f8cc593b530b42b8b2573b945b405793 2024-11-19T17:49:02,994 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/A of 42eb2452c7f2a30803f85b3a18651fb5 into f8cc593b530b42b8b2573b945b405793(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:02,994 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:02,994 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/A, priority=13, startTime=1732038542522; duration=0sec 2024-11-19T17:49:02,994 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:02,994 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:A 2024-11-19T17:49:03,199 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/3bdbceaefe9b4529a6936a3221eb588e 2024-11-19T17:49:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/671b301a39c94f1c820b6ff747791d83 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/671b301a39c94f1c820b6ff747791d83 2024-11-19T17:49:03,207 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/671b301a39c94f1c820b6ff747791d83, entries=150, sequenceid=403, filesize=30.5 K 2024-11-19T17:49:03,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ef599094dbaa4d8785327803c9694b60 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ef599094dbaa4d8785327803c9694b60 2024-11-19T17:49:03,212 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ef599094dbaa4d8785327803c9694b60, entries=150, sequenceid=403, filesize=12.0 K 2024-11-19T17:49:03,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/3bdbceaefe9b4529a6936a3221eb588e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3bdbceaefe9b4529a6936a3221eb588e 2024-11-19T17:49:03,217 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3bdbceaefe9b4529a6936a3221eb588e, entries=150, sequenceid=403, filesize=12.0 K 2024-11-19T17:49:03,218 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=20.13 KB/20610 for 42eb2452c7f2a30803f85b3a18651fb5 in 561ms, sequenceid=403, compaction requested=false 2024-11-19T17:49:03,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:03,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:03,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-19T17:49:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-19T17:49:03,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T17:49:03,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 867 msec 2024-11-19T17:49:03,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 872 msec 2024-11-19T17:49:03,382 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/2dc37926436547cf99c252a33f2cfdae as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2dc37926436547cf99c252a33f2cfdae 2024-11-19T17:49:03,385 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42eb2452c7f2a30803f85b3a18651fb5/C of 42eb2452c7f2a30803f85b3a18651fb5 into 2dc37926436547cf99c252a33f2cfdae(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:03,385 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42eb2452c7f2a30803f85b3a18651fb5: 2024-11-19T17:49:03,385 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., storeName=42eb2452c7f2a30803f85b3a18651fb5/C, priority=13, startTime=1732038542523; duration=0sec 2024-11-19T17:49:03,386 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:03,386 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42eb2452c7f2a30803f85b3a18651fb5:C 2024-11-19T17:49:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:49:03,489 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:07,920 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-19T17:49:07,920 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:07,920 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:07,920 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:07,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:07,951 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-19T17:49:07,951 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:07,951 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:07,951 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:07,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3593 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3601 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1552 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4656 rows 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1560 2024-11-19T17:49:07,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4678 rows 2024-11-19T17:49:07,952 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:49:07,952 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:49:07,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:07,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:07,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:07,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:49:07,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:49:07,959 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038547959"}]},"ts":"1732038547959"} 2024-11-19T17:49:07,962 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:49:07,962 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:49:07,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:49:07,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, UNASSIGN}] 2024-11-19T17:49:07,966 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, UNASSIGN 2024-11-19T17:49:07,967 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=42eb2452c7f2a30803f85b3a18651fb5, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:07,968 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41481 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=26d3106abcfe,36839,1732038481522, table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T17:49:07,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, UNASSIGN because future has completed 2024-11-19T17:49:07,969 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:49:07,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:49:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:49:08,122 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:08,122 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:49:08,122 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 42eb2452c7f2a30803f85b3a18651fb5, disabling compactions & flushes 2024-11-19T17:49:08,122 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:08,122 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:08,122 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. after waiting 0 ms 2024-11-19T17:49:08,122 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:08,122 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 42eb2452c7f2a30803f85b3a18651fb5 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T17:49:08,132 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d967370f7bce478b8ec7cef88f3ad2f6_42eb2452c7f2a30803f85b3a18651fb5 is 50, key is test_row_0/A:col10/1732038542858/Put/seqid=0 2024-11-19T17:49:08,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742079_1255 (size=12454) 2024-11-19T17:49:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:49:08,539 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:08,546 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d967370f7bce478b8ec7cef88f3ad2f6_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d967370f7bce478b8ec7cef88f3ad2f6_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:08,548 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/9be34829638c4b4799b4e26263b6a85d, store: [table=TestAcidGuarantees family=A region=42eb2452c7f2a30803f85b3a18651fb5] 2024-11-19T17:49:08,548 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/9be34829638c4b4799b4e26263b6a85d is 175, key is test_row_0/A:col10/1732038542858/Put/seqid=0 2024-11-19T17:49:08,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742080_1256 (size=31255) 2024-11-19T17:49:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:49:08,954 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/9be34829638c4b4799b4e26263b6a85d 2024-11-19T17:49:08,969 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ba04177029b24ab6a6335bc8ea4e9ac3 is 50, key is test_row_0/B:col10/1732038542858/Put/seqid=0 2024-11-19T17:49:08,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742081_1257 (size=12301) 2024-11-19T17:49:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:49:09,374 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ba04177029b24ab6a6335bc8ea4e9ac3 2024-11-19T17:49:09,385 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/7b85624d9a2545de905430a6c3ea4a13 is 50, key is test_row_0/C:col10/1732038542858/Put/seqid=0 2024-11-19T17:49:09,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742082_1258 (size=12301) 2024-11-19T17:49:09,791 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/7b85624d9a2545de905430a6c3ea4a13 2024-11-19T17:49:09,802 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/A/9be34829638c4b4799b4e26263b6a85d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/9be34829638c4b4799b4e26263b6a85d 2024-11-19T17:49:09,806 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/9be34829638c4b4799b4e26263b6a85d, entries=150, sequenceid=414, filesize=30.5 K 2024-11-19T17:49:09,806 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/B/ba04177029b24ab6a6335bc8ea4e9ac3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba04177029b24ab6a6335bc8ea4e9ac3 2024-11-19T17:49:09,810 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba04177029b24ab6a6335bc8ea4e9ac3, entries=150, sequenceid=414, filesize=12.0 K 2024-11-19T17:49:09,811 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/.tmp/C/7b85624d9a2545de905430a6c3ea4a13 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b85624d9a2545de905430a6c3ea4a13 2024-11-19T17:49:09,815 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b85624d9a2545de905430a6c3ea4a13, entries=150, sequenceid=414, filesize=12.0 K 2024-11-19T17:49:09,815 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 42eb2452c7f2a30803f85b3a18651fb5 in 1693ms, sequenceid=414, compaction requested=true 2024-11-19T17:49:09,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/6322e9245237414fa9aa1966792d2ed8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/75db6c31ac0d4b04ab1a8907a40c2c3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/246f1696bac34bedb63801757ada2513, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5149b4c4efb34de48e56407acb7b3304, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/edd57cb4a1814d38a1c28a650c5dfcc6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e46c792fdda045e781baa17a2d26ddb9, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c756f554cb3240b3acda3524ba1684bc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5da6107e45f44e7c905db30ff6a69ab0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e561b311b2544b20971505530f97e70a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/606e029731924bf990499a122ab13fad, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/a01d9051cb574f9487534caf0f774492, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/d621824055044b00bd46969071822f8c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/2591a407be79422890555c96864e0fad, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/bd3766a6eb2248e2b913d2baa83c350d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/25720389e6bb4fd9b9b8aae762b79311, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/7ee45a0735d5497297eae38e2a9dd834, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/807bf05c139343fb8f35ced8ce3c15ab, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/1208d0264bd743c8ae685e316a7cbf77, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/8e90e04d1b664792a8a79621ae2f7a6e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/20ee1f072d5a4ffba922d85cd96e6404, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5e4f8a4013924c0fbfcd9278e0dd3691, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c5b72449b94b40bf94d75bf737f289a0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5be5e45e4a53429795755812fbf82204, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5696ec70eed34000bc38d167f07c98aa, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/90c31f3275de48f6aee208e1aeb73e86, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5df63bf5a66d425b8d3c471b944a08f5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/95458fa3d2024ea7a344bea0a8c7d3d0] to archive 2024-11-19T17:49:09,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:49:09,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/6322e9245237414fa9aa1966792d2ed8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/6322e9245237414fa9aa1966792d2ed8 2024-11-19T17:49:09,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/75db6c31ac0d4b04ab1a8907a40c2c3d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/75db6c31ac0d4b04ab1a8907a40c2c3d 2024-11-19T17:49:09,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/246f1696bac34bedb63801757ada2513 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/246f1696bac34bedb63801757ada2513 2024-11-19T17:49:09,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5149b4c4efb34de48e56407acb7b3304 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5149b4c4efb34de48e56407acb7b3304 2024-11-19T17:49:09,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/edd57cb4a1814d38a1c28a650c5dfcc6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/edd57cb4a1814d38a1c28a650c5dfcc6 2024-11-19T17:49:09,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e46c792fdda045e781baa17a2d26ddb9 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e46c792fdda045e781baa17a2d26ddb9 2024-11-19T17:49:09,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c756f554cb3240b3acda3524ba1684bc to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c756f554cb3240b3acda3524ba1684bc 2024-11-19T17:49:09,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5da6107e45f44e7c905db30ff6a69ab0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5da6107e45f44e7c905db30ff6a69ab0 2024-11-19T17:49:09,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e561b311b2544b20971505530f97e70a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/e561b311b2544b20971505530f97e70a 2024-11-19T17:49:09,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/606e029731924bf990499a122ab13fad to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/606e029731924bf990499a122ab13fad 2024-11-19T17:49:09,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/a01d9051cb574f9487534caf0f774492 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/a01d9051cb574f9487534caf0f774492 2024-11-19T17:49:09,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/d621824055044b00bd46969071822f8c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/d621824055044b00bd46969071822f8c 2024-11-19T17:49:09,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/2591a407be79422890555c96864e0fad to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/2591a407be79422890555c96864e0fad 2024-11-19T17:49:09,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/bd3766a6eb2248e2b913d2baa83c350d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/bd3766a6eb2248e2b913d2baa83c350d 2024-11-19T17:49:09,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/25720389e6bb4fd9b9b8aae762b79311 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/25720389e6bb4fd9b9b8aae762b79311 2024-11-19T17:49:09,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/7ee45a0735d5497297eae38e2a9dd834 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/7ee45a0735d5497297eae38e2a9dd834 2024-11-19T17:49:09,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/807bf05c139343fb8f35ced8ce3c15ab to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/807bf05c139343fb8f35ced8ce3c15ab 2024-11-19T17:49:09,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/1208d0264bd743c8ae685e316a7cbf77 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/1208d0264bd743c8ae685e316a7cbf77 2024-11-19T17:49:09,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/8e90e04d1b664792a8a79621ae2f7a6e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/8e90e04d1b664792a8a79621ae2f7a6e 2024-11-19T17:49:09,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/20ee1f072d5a4ffba922d85cd96e6404 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/20ee1f072d5a4ffba922d85cd96e6404 2024-11-19T17:49:09,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5e4f8a4013924c0fbfcd9278e0dd3691 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5e4f8a4013924c0fbfcd9278e0dd3691 2024-11-19T17:49:09,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c5b72449b94b40bf94d75bf737f289a0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/c5b72449b94b40bf94d75bf737f289a0 2024-11-19T17:49:09,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5be5e45e4a53429795755812fbf82204 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5be5e45e4a53429795755812fbf82204 2024-11-19T17:49:09,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5696ec70eed34000bc38d167f07c98aa to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5696ec70eed34000bc38d167f07c98aa 2024-11-19T17:49:09,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/90c31f3275de48f6aee208e1aeb73e86 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/90c31f3275de48f6aee208e1aeb73e86 2024-11-19T17:49:09,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5df63bf5a66d425b8d3c471b944a08f5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/5df63bf5a66d425b8d3c471b944a08f5 2024-11-19T17:49:09,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/95458fa3d2024ea7a344bea0a8c7d3d0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/95458fa3d2024ea7a344bea0a8c7d3d0 2024-11-19T17:49:09,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d7d1a4afff5c412898bf19e6ce05e132, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/31063628d54845948755e3ed91f9dcfd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a4bd45351dcd420bb6f945af362dcc20, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a28697f1b5dd4f58bbfff538d10d8113, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dad94aa1aff147538f6872be7b78406b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a88ae4f49dea4a508623d2ed7bebd1cc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/88e3e7446c324fd4b598c8494e39bc41, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/137f6f37f2ef46e89f2168762cd5a882, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba90ec56c6ee435693e34d17519b0d97, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/b976604cea204f57aeac095ed7634ec7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/640752ed8d4a427581964dd216c59bf1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/95146989d82c4510befa4ae9c19343b4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/fb2f21ff863b4577a40efa2d6f96be49, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56b8f24d67ff4120bd8bb1fe3b9ad90a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a26f1c5909b4457d8a068b3cb014a1b6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ea6ef955f17646f6a6661e0a1086c317, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/7a5ff037c15c45bb90c6cb3226f187fc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d5a446376bda4eaf8bca9de42b03b0bf, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/1e69a1c0dbf842fba5f2ee3704d8f3b3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d193e34f29414545ab651b7419351400, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/c76e288d22ab4b1b92fa21b6225ab616, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/078b130d2d434d58ae18d4f18e6abac6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dcff1415d7d74c318bba3fb48af80f91, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/4c9f3b4b9ee8410591bfc8a58370d886, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56a7430a65ab4570b44e0add8608886a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/8dcf1730dc68405db84c3bff4f438a46, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/45bc4684c8554a798d7d695e4c259dd3] to archive 2024-11-19T17:49:09,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:49:09,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d7d1a4afff5c412898bf19e6ce05e132 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d7d1a4afff5c412898bf19e6ce05e132 2024-11-19T17:49:09,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/31063628d54845948755e3ed91f9dcfd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/31063628d54845948755e3ed91f9dcfd 2024-11-19T17:49:09,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a4bd45351dcd420bb6f945af362dcc20 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a4bd45351dcd420bb6f945af362dcc20 2024-11-19T17:49:09,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a28697f1b5dd4f58bbfff538d10d8113 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a28697f1b5dd4f58bbfff538d10d8113 2024-11-19T17:49:09,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dad94aa1aff147538f6872be7b78406b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dad94aa1aff147538f6872be7b78406b 2024-11-19T17:49:09,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a88ae4f49dea4a508623d2ed7bebd1cc to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a88ae4f49dea4a508623d2ed7bebd1cc 2024-11-19T17:49:09,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/88e3e7446c324fd4b598c8494e39bc41 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/88e3e7446c324fd4b598c8494e39bc41 2024-11-19T17:49:09,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/137f6f37f2ef46e89f2168762cd5a882 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/137f6f37f2ef46e89f2168762cd5a882 2024-11-19T17:49:09,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba90ec56c6ee435693e34d17519b0d97 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba90ec56c6ee435693e34d17519b0d97 2024-11-19T17:49:09,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/b976604cea204f57aeac095ed7634ec7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/b976604cea204f57aeac095ed7634ec7 2024-11-19T17:49:09,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/640752ed8d4a427581964dd216c59bf1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/640752ed8d4a427581964dd216c59bf1 2024-11-19T17:49:09,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/95146989d82c4510befa4ae9c19343b4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/95146989d82c4510befa4ae9c19343b4 2024-11-19T17:49:09,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/fb2f21ff863b4577a40efa2d6f96be49 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/fb2f21ff863b4577a40efa2d6f96be49 2024-11-19T17:49:09,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56b8f24d67ff4120bd8bb1fe3b9ad90a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56b8f24d67ff4120bd8bb1fe3b9ad90a 2024-11-19T17:49:09,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a26f1c5909b4457d8a068b3cb014a1b6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a26f1c5909b4457d8a068b3cb014a1b6 2024-11-19T17:49:09,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ea6ef955f17646f6a6661e0a1086c317 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ea6ef955f17646f6a6661e0a1086c317 2024-11-19T17:49:09,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/7a5ff037c15c45bb90c6cb3226f187fc to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/7a5ff037c15c45bb90c6cb3226f187fc 2024-11-19T17:49:09,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d5a446376bda4eaf8bca9de42b03b0bf to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d5a446376bda4eaf8bca9de42b03b0bf 2024-11-19T17:49:09,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/1e69a1c0dbf842fba5f2ee3704d8f3b3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/1e69a1c0dbf842fba5f2ee3704d8f3b3 2024-11-19T17:49:09,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d193e34f29414545ab651b7419351400 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/d193e34f29414545ab651b7419351400 2024-11-19T17:49:09,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/c76e288d22ab4b1b92fa21b6225ab616 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/c76e288d22ab4b1b92fa21b6225ab616 2024-11-19T17:49:09,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/078b130d2d434d58ae18d4f18e6abac6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/078b130d2d434d58ae18d4f18e6abac6 2024-11-19T17:49:09,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dcff1415d7d74c318bba3fb48af80f91 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/dcff1415d7d74c318bba3fb48af80f91 2024-11-19T17:49:09,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/4c9f3b4b9ee8410591bfc8a58370d886 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/4c9f3b4b9ee8410591bfc8a58370d886 2024-11-19T17:49:09,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56a7430a65ab4570b44e0add8608886a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/56a7430a65ab4570b44e0add8608886a 2024-11-19T17:49:09,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/8dcf1730dc68405db84c3bff4f438a46 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/8dcf1730dc68405db84c3bff4f438a46 2024-11-19T17:49:09,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/45bc4684c8554a798d7d695e4c259dd3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/45bc4684c8554a798d7d695e4c259dd3 2024-11-19T17:49:09,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ad9fcfbc55b94d54bc90ee0d807d61b1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/f96a681d27f74ad99eb750f7c2e2f670, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b88587e2558465fba4a241b03937aae, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e432050e88834c74916d53620111a259, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c63c5691fc6743448ef3918b61530729, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/66ac4a9b49324fba97d8c954c061a31e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/441233117a13433f8ca6931a66c22fed, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/92e487ee793440118a37587a4c0546f2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/895008d86d1a48fbaa61da807eb5fc30, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d4ffd27020974c3088be25aa1653c233, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/8ceb7e2353f14d7e952c0fa5f9ec911c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c64b17a9817a46c1b344040eaf24f3cc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/81833bdcb772421390a42eb021c06a2b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/b24f8bc7a409427dbee5ff1b52e4ce00, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/9db754fac5ee4a94874ba5a39b117b1f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/cf5808343edc4e02aa42bedfda0567b4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/270c799c85754e809d8a2b8db71efabd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/dc029795a9f246e99bd94ecc795e8548, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d1f86817e5f742ceb75d961d1053cfe0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/a4d4c892df85462ea92a38fa9076d08f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2f02ebddc0034531a13d875c4727a45c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e56c06922f0e4114ba2f0d05c1ce9050, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/648aa403860643218017da3a34ad71c0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c6f97cf346724c1e9fdff8060a0f77c5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3f636fb7dd1348cb9d8a8d2abf330468, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/61624145cf984408b53f6efe9f47a770] to archive 2024-11-19T17:49:09,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:49:09,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ad9fcfbc55b94d54bc90ee0d807d61b1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ad9fcfbc55b94d54bc90ee0d807d61b1 2024-11-19T17:49:09,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/f96a681d27f74ad99eb750f7c2e2f670 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/f96a681d27f74ad99eb750f7c2e2f670 2024-11-19T17:49:09,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b88587e2558465fba4a241b03937aae to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b88587e2558465fba4a241b03937aae 2024-11-19T17:49:09,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e432050e88834c74916d53620111a259 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e432050e88834c74916d53620111a259 2024-11-19T17:49:09,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c63c5691fc6743448ef3918b61530729 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c63c5691fc6743448ef3918b61530729 2024-11-19T17:49:09,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/66ac4a9b49324fba97d8c954c061a31e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/66ac4a9b49324fba97d8c954c061a31e 2024-11-19T17:49:09,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/441233117a13433f8ca6931a66c22fed to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/441233117a13433f8ca6931a66c22fed 2024-11-19T17:49:09,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/ed1994e6406b4e8ba8bdfb5a6dfe1b4e 2024-11-19T17:49:09,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/92e487ee793440118a37587a4c0546f2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/92e487ee793440118a37587a4c0546f2 2024-11-19T17:49:09,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/895008d86d1a48fbaa61da807eb5fc30 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/895008d86d1a48fbaa61da807eb5fc30 2024-11-19T17:49:09,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d4ffd27020974c3088be25aa1653c233 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d4ffd27020974c3088be25aa1653c233 2024-11-19T17:49:09,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/8ceb7e2353f14d7e952c0fa5f9ec911c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/8ceb7e2353f14d7e952c0fa5f9ec911c 2024-11-19T17:49:09,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c64b17a9817a46c1b344040eaf24f3cc to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c64b17a9817a46c1b344040eaf24f3cc 2024-11-19T17:49:09,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/81833bdcb772421390a42eb021c06a2b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/81833bdcb772421390a42eb021c06a2b 2024-11-19T17:49:09,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/b24f8bc7a409427dbee5ff1b52e4ce00 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/b24f8bc7a409427dbee5ff1b52e4ce00 2024-11-19T17:49:09,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/9db754fac5ee4a94874ba5a39b117b1f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/9db754fac5ee4a94874ba5a39b117b1f 2024-11-19T17:49:09,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/cf5808343edc4e02aa42bedfda0567b4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/cf5808343edc4e02aa42bedfda0567b4 2024-11-19T17:49:09,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/270c799c85754e809d8a2b8db71efabd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/270c799c85754e809d8a2b8db71efabd 2024-11-19T17:49:09,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/dc029795a9f246e99bd94ecc795e8548 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/dc029795a9f246e99bd94ecc795e8548 2024-11-19T17:49:09,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d1f86817e5f742ceb75d961d1053cfe0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/d1f86817e5f742ceb75d961d1053cfe0 2024-11-19T17:49:09,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/a4d4c892df85462ea92a38fa9076d08f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/a4d4c892df85462ea92a38fa9076d08f 2024-11-19T17:49:09,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2f02ebddc0034531a13d875c4727a45c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2f02ebddc0034531a13d875c4727a45c 2024-11-19T17:49:09,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e56c06922f0e4114ba2f0d05c1ce9050 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/e56c06922f0e4114ba2f0d05c1ce9050 2024-11-19T17:49:09,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/648aa403860643218017da3a34ad71c0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/648aa403860643218017da3a34ad71c0 2024-11-19T17:49:09,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c6f97cf346724c1e9fdff8060a0f77c5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/c6f97cf346724c1e9fdff8060a0f77c5 2024-11-19T17:49:09,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3f636fb7dd1348cb9d8a8d2abf330468 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3f636fb7dd1348cb9d8a8d2abf330468 2024-11-19T17:49:09,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/61624145cf984408b53f6efe9f47a770 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/61624145cf984408b53f6efe9f47a770 2024-11-19T17:49:09,898 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/recovered.edits/417.seqid, newMaxSeqId=417, maxSeqId=4 2024-11-19T17:49:09,899 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. 2024-11-19T17:49:09,899 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 42eb2452c7f2a30803f85b3a18651fb5: Waiting for close lock at 1732038548122Running coprocessor pre-close hooks at 1732038548122Disabling compacts and flushes for region at 1732038548122Disabling writes for close at 1732038548122Obtaining lock to block concurrent updates at 1732038548122Preparing flush snapshotting stores in 42eb2452c7f2a30803f85b3a18651fb5 at 1732038548122Finished memstore snapshotting TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732038548122Flushing stores of TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5. at 1732038548123 (+1 ms)Flushing 42eb2452c7f2a30803f85b3a18651fb5/A: creating writer at 1732038548123Flushing mob file 42eb2452c7f2a30803f85b3a18651fb5/A: appending metadata at 1732038548131 (+8 ms)Flushing mob file 42eb2452c7f2a30803f85b3a18651fb5/A: closing flushed file at 1732038548132 (+1 ms)Flushing 42eb2452c7f2a30803f85b3a18651fb5/A: appending metadata at 1732038548548 (+416 ms)Flushing 42eb2452c7f2a30803f85b3a18651fb5/A: closing flushed file at 1732038548548Flushing 42eb2452c7f2a30803f85b3a18651fb5/B: creating writer at 1732038548963 (+415 ms)Flushing 42eb2452c7f2a30803f85b3a18651fb5/B: appending metadata at 1732038548968 (+5 ms)Flushing 42eb2452c7f2a30803f85b3a18651fb5/B: closing flushed file at 1732038548969 (+1 ms)Flushing 42eb2452c7f2a30803f85b3a18651fb5/C: creating writer at 1732038549381 (+412 ms)Flushing 42eb2452c7f2a30803f85b3a18651fb5/C: appending metadata at 1732038549385 (+4 ms)Flushing 42eb2452c7f2a30803f85b3a18651fb5/C: closing flushed file at 1732038549385Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3831018f: reopening flushed file at 1732038549801 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c01a738: reopening flushed file at 1732038549806 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@598c30b3: reopening flushed file at 1732038549810 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 42eb2452c7f2a30803f85b3a18651fb5 in 1693ms, sequenceid=414, compaction requested=true at 1732038549815 (+5 ms)Writing region close event to WAL at 1732038549896 (+81 ms)Running coprocessor post-close hooks at 1732038549899 (+3 ms)Closed at 1732038549899 2024-11-19T17:49:09,901 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:09,901 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=42eb2452c7f2a30803f85b3a18651fb5, regionState=CLOSED 2024-11-19T17:49:09,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:49:09,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T17:49:09,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 42eb2452c7f2a30803f85b3a18651fb5, server=26d3106abcfe,36839,1732038481522 in 1.9340 sec 2024-11-19T17:49:09,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-19T17:49:09,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42eb2452c7f2a30803f85b3a18651fb5, UNASSIGN in 1.9390 sec 2024-11-19T17:49:09,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T17:49:09,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9420 sec 2024-11-19T17:49:09,908 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038549907"}]},"ts":"1732038549907"} 2024-11-19T17:49:09,909 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:49:09,909 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:49:09,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9550 sec 2024-11-19T17:49:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:49:10,109 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:10,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:49:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:10,112 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T17:49:10,113 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:10,118 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,122 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/recovered.edits] 2024-11-19T17:49:10,126 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/671b301a39c94f1c820b6ff747791d83 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/671b301a39c94f1c820b6ff747791d83 2024-11-19T17:49:10,128 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/9be34829638c4b4799b4e26263b6a85d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/9be34829638c4b4799b4e26263b6a85d 2024-11-19T17:49:10,130 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/f8cc593b530b42b8b2573b945b405793 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/A/f8cc593b530b42b8b2573b945b405793 2024-11-19T17:49:10,133 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a9dcb412c1634276a0a77b04940ddacc to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/a9dcb412c1634276a0a77b04940ddacc 2024-11-19T17:49:10,135 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba04177029b24ab6a6335bc8ea4e9ac3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ba04177029b24ab6a6335bc8ea4e9ac3 2024-11-19T17:49:10,137 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ef599094dbaa4d8785327803c9694b60 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/B/ef599094dbaa4d8785327803c9694b60 2024-11-19T17:49:10,141 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2dc37926436547cf99c252a33f2cfdae to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/2dc37926436547cf99c252a33f2cfdae 2024-11-19T17:49:10,143 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3bdbceaefe9b4529a6936a3221eb588e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/3bdbceaefe9b4529a6936a3221eb588e 2024-11-19T17:49:10,145 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b85624d9a2545de905430a6c3ea4a13 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/C/7b85624d9a2545de905430a6c3ea4a13 2024-11-19T17:49:10,149 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/recovered.edits/417.seqid to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5/recovered.edits/417.seqid 2024-11-19T17:49:10,150 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,150 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:49:10,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:49:10,151 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T17:49:10,154 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d46869cc063440a8a9a1bdfa5dd114f_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d46869cc063440a8a9a1bdfa5dd114f_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,155 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911d30a6350684cdc9335e366ab9a0be8_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911d30a6350684cdc9335e366ab9a0be8_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119123d86b4690146749c356642531eb8a3_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119123d86b4690146749c356642531eb8a3_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,157 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fe496a6100745a0b9e5651f229c64aa_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fe496a6100745a0b9e5651f229c64aa_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,158 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119367948eead3343f981c20525befab04f_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119367948eead3343f981c20525befab04f_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941f18b30198e4ffe8732f149f9249fef_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941f18b30198e4ffe8732f149f9249fef_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,160 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195a59353b3f03423caa50c4599d2791bc_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195a59353b3f03423caa50c4599d2791bc_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,161 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ad7382647ff44519f49c44ea2127a0d_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ad7382647ff44519f49c44ea2127a0d_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,161 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197f4af5285e1f43e8aa7603fb188fc6a4_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197f4af5285e1f43e8aa7603fb188fc6a4_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111998d8b4721267478a912d1bdc74f63de9_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111998d8b4721267478a912d1bdc74f63de9_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,163 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111999844ed489f04ca89b5227ae90d65c4c_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111999844ed489f04ca89b5227ae90d65c4c_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,164 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a212ee8e8f0f422e90127a073308ad0b_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a212ee8e8f0f422e90127a073308ad0b_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,164 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a7486023243c42399e2073a52769a44c_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a7486023243c42399e2073a52769a44c_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,165 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad123d6925244540ac6e97c61afc73b9_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad123d6925244540ac6e97c61afc73b9_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,166 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8419b5d26804e7fae979ec6350a2b2d_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8419b5d26804e7fae979ec6350a2b2d_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,167 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0a06242e5b549baa7061549ff118f28_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0a06242e5b549baa7061549ff118f28_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,168 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ca3d879cacd54061ab294ec7c84e3e2e_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ca3d879cacd54061ab294ec7c84e3e2e_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,168 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d5e8f6714cdf400bb77e2836dc6670ae_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d5e8f6714cdf400bb77e2836dc6670ae_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,169 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d967370f7bce478b8ec7cef88f3ad2f6_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d967370f7bce478b8ec7cef88f3ad2f6_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,170 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dee19745a7554a9491020efde6c46545_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dee19745a7554a9491020efde6c46545_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,171 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eb942bd032e14dbfaec2d4152446296f_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eb942bd032e14dbfaec2d4152446296f_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,172 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119efb388da5ccb4f57a78d28e7c1dc3d10_42eb2452c7f2a30803f85b3a18651fb5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119efb388da5ccb4f57a78d28e7c1dc3d10_42eb2452c7f2a30803f85b3a18651fb5 2024-11-19T17:49:10,172 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:49:10,174 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:10,176 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:49:10,177 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:49:10,178 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:10,178 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:49:10,179 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732038550178"}]},"ts":"9223372036854775807"} 2024-11-19T17:49:10,180 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:49:10,180 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 42eb2452c7f2a30803f85b3a18651fb5, NAME => 'TestAcidGuarantees,,1732038519641.42eb2452c7f2a30803f85b3a18651fb5.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:49:10,180 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:49:10,180 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732038550180"}]},"ts":"9223372036854775807"} 2024-11-19T17:49:10,182 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:49:10,182 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:10,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-11-19T17:49:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T17:49:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:49:10,219 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:10,229 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 237) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1036582635_22 at /127.0.0.1:45900 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=568 (was 708), ProcessCount=11 (was 11), AvailableMemoryMB=9611 (was 9722) 2024-11-19T17:49:10,237 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=568, ProcessCount=11, AvailableMemoryMB=9611 2024-11-19T17:49:10,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:49:10,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:49:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:10,240 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:49:10,240 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:10,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-19T17:49:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T17:49:10,241 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:49:10,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742083_1259 (size=959) 2024-11-19T17:49:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T17:49:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T17:49:10,652 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d 2024-11-19T17:49:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742084_1260 (size=53) 2024-11-19T17:49:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T17:49:10,976 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:49:11,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:49:11,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f7aa8054d507e358fea7889f8e6a6f0a, disabling compactions & flushes 2024-11-19T17:49:11,062 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:11,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:11,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. after waiting 0 ms 2024-11-19T17:49:11,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:11,062 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:11,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f7aa8054d507e358fea7889f8e6a6f0a: Waiting for close lock at 1732038551062Disabling compacts and flushes for region at 1732038551062Disabling writes for close at 1732038551062Writing region close event to WAL at 1732038551062Closed at 1732038551062 2024-11-19T17:49:11,063 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:49:11,064 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732038551064"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732038551064"}]},"ts":"1732038551064"} 2024-11-19T17:49:11,065 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:49:11,066 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:49:11,067 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038551066"}]},"ts":"1732038551066"} 2024-11-19T17:49:11,068 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:49:11,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, ASSIGN}] 2024-11-19T17:49:11,070 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, ASSIGN 2024-11-19T17:49:11,070 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, ASSIGN; state=OFFLINE, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=false 2024-11-19T17:49:11,221 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f7aa8054d507e358fea7889f8e6a6f0a, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:11,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, ASSIGN because future has completed 2024-11-19T17:49:11,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:49:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T17:49:11,380 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:11,380 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:49:11,381 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,381 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:49:11,381 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,382 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,384 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,386 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7aa8054d507e358fea7889f8e6a6f0a columnFamilyName A 2024-11-19T17:49:11,387 DEBUG [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:11,388 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] regionserver.HStore(327): Store=f7aa8054d507e358fea7889f8e6a6f0a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:11,388 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,390 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7aa8054d507e358fea7889f8e6a6f0a columnFamilyName B 2024-11-19T17:49:11,390 DEBUG [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:11,391 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] regionserver.HStore(327): Store=f7aa8054d507e358fea7889f8e6a6f0a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:11,391 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,393 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7aa8054d507e358fea7889f8e6a6f0a columnFamilyName C 2024-11-19T17:49:11,393 DEBUG [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:11,393 INFO [StoreOpener-f7aa8054d507e358fea7889f8e6a6f0a-1 {}] regionserver.HStore(327): Store=f7aa8054d507e358fea7889f8e6a6f0a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:11,393 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,394 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,394 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,396 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,396 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,396 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:49:11,398 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,400 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:49:11,400 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened f7aa8054d507e358fea7889f8e6a6f0a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67428458, jitterRate=0.004762321710586548}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:49:11,400 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:11,402 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for f7aa8054d507e358fea7889f8e6a6f0a: Running coprocessor pre-open hook at 1732038551382Writing region info on filesystem at 1732038551382Initializing all the Stores at 1732038551383 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038551383Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038551384 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038551384Cleaning up temporary data from old regions at 1732038551396 (+12 ms)Running coprocessor post-open hooks at 1732038551400 (+4 ms)Region opened successfully at 1732038551401 (+1 ms) 2024-11-19T17:49:11,402 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., pid=67, masterSystemTime=1732038551376 2024-11-19T17:49:11,404 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:11,405 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:11,405 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f7aa8054d507e358fea7889f8e6a6f0a, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:11,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:49:11,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T17:49:11,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 in 185 msec 2024-11-19T17:49:11,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-19T17:49:11,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, ASSIGN in 341 msec 2024-11-19T17:49:11,413 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:49:11,413 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038551413"}]},"ts":"1732038551413"} 2024-11-19T17:49:11,414 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:49:11,415 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:49:11,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-19T17:49:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T17:49:12,399 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:12,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,402 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159331c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752e07c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,405 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,408 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:49:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,41481,1732038480627 2024-11-19T17:49:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e52840c 2024-11-19T17:49:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:49:12,410 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:49:12,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,412 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d4939a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,414 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21363cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,417 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b732cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11694ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,421 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d5076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8fa8e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f93b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eddb050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,429 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2629c2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,431 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b8116f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,434 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3ec183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,435 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b91ec2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,438 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f74eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,440 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20aa5a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,444 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d9af43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,446 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da6fa17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,450 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,451 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,453 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f8f745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:12,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:12,456 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519d7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:12,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:12,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:12,458 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3ff71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:12,459 DEBUG [Thread-1214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,460 DEBUG [Thread-1216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,464 DEBUG [Thread-1218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:12,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,466 DEBUG [Thread-1201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T17:49:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:49:12,468 DEBUG [Thread-1201 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,470 DEBUG [Thread-1203 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,470 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:12,471 DEBUG [Thread-1212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,471 DEBUG [Thread-1203 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,471 DEBUG [Thread-1212 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,471 DEBUG [Thread-1203 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,471 DEBUG [Thread-1212 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,472 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:12,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,476 DEBUG [Thread-1220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,477 DEBUG [Thread-1220 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,479 DEBUG [Thread-1220 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,496 DEBUG [Thread-1205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,499 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:12,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,513 DEBUG [Thread-1207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,524 DEBUG [Thread-1209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:12,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:12,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:49:12,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/49b52b0be6a0439395597a9e94fc0341 is 50, key is test_row_0/A:col10/1732038552496/Put/seqid=0 2024-11-19T17:49:12,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038612549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038612553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038612556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038612556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038612556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:49:12,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742085_1261 (size=12001) 2024-11-19T17:49:12,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T17:49:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:12,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038612669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038612669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038612679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038612679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038612679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T17:49:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:12,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:12,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:12,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:49:12,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038612878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038612878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038612889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038612889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038612889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:12,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T17:49:12,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:12,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:12,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:12,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:12,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/49b52b0be6a0439395597a9e94fc0341 2024-11-19T17:49:13,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/59a66c219831455bbecd474cbde0817d is 50, key is test_row_0/B:col10/1732038552496/Put/seqid=0 2024-11-19T17:49:13,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T17:49:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:49:13,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742086_1262 (size=12001) 2024-11-19T17:49:13,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038613188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038613189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038613199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038613199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038613199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T17:49:13,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:13,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T17:49:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/59a66c219831455bbecd474cbde0817d 2024-11-19T17:49:13,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T17:49:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:13,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/e09052e0239c46878990b63dc461cad7 is 50, key is test_row_0/C:col10/1732038552496/Put/seqid=0 2024-11-19T17:49:13,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742087_1263 (size=12001) 2024-11-19T17:49:13,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/e09052e0239c46878990b63dc461cad7 2024-11-19T17:49:13,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/49b52b0be6a0439395597a9e94fc0341 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/49b52b0be6a0439395597a9e94fc0341 2024-11-19T17:49:13,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/49b52b0be6a0439395597a9e94fc0341, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T17:49:13,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/59a66c219831455bbecd474cbde0817d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/59a66c219831455bbecd474cbde0817d 2024-11-19T17:49:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:49:13,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/59a66c219831455bbecd474cbde0817d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T17:49:13,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/e09052e0239c46878990b63dc461cad7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e09052e0239c46878990b63dc461cad7 2024-11-19T17:49:13,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e09052e0239c46878990b63dc461cad7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T17:49:13,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for f7aa8054d507e358fea7889f8e6a6f0a in 1119ms, sequenceid=12, compaction requested=false 2024-11-19T17:49:13,619 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:49:13,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:13,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T17:49:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,702 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T17:49:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:13,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/d2a97f695fbc44d9805d82b74327db3c is 50, key is test_row_0/A:col10/1732038553701/Put/seqid=0 2024-11-19T17:49:13,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742088_1264 (size=14341) 2024-11-19T17:49:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038613709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,715 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/d2a97f695fbc44d9805d82b74327db3c 2024-11-19T17:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038613713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038613713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038613714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038613714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8b73216d25c54d5b84bf7bb9586e9e17 is 50, key is test_row_0/B:col10/1732038553701/Put/seqid=0 2024-11-19T17:49:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742089_1265 (size=12001) 2024-11-19T17:49:13,766 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8b73216d25c54d5b84bf7bb9586e9e17 2024-11-19T17:49:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/39afe135b6924d74b4d7390ca8af1cd2 is 50, key is test_row_0/C:col10/1732038553701/Put/seqid=0 2024-11-19T17:49:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742090_1266 (size=12001) 2024-11-19T17:49:13,799 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/39afe135b6924d74b4d7390ca8af1cd2 2024-11-19T17:49:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/d2a97f695fbc44d9805d82b74327db3c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2a97f695fbc44d9805d82b74327db3c 2024-11-19T17:49:13,815 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2a97f695fbc44d9805d82b74327db3c, entries=200, sequenceid=40, filesize=14.0 K 2024-11-19T17:49:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8b73216d25c54d5b84bf7bb9586e9e17 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8b73216d25c54d5b84bf7bb9586e9e17 2024-11-19T17:49:13,823 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8b73216d25c54d5b84bf7bb9586e9e17, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T17:49:13,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/39afe135b6924d74b4d7390ca8af1cd2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39afe135b6924d74b4d7390ca8af1cd2 2024-11-19T17:49:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038613829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,830 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39afe135b6924d74b4d7390ca8af1cd2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T17:49:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038613829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,831 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for f7aa8054d507e358fea7889f8e6a6f0a in 129ms, sequenceid=40, compaction requested=false 2024-11-19T17:49:13,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:13,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:13,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T17:49:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T17:49:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:13,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:49:13,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T17:49:13,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3600 sec 2024-11-19T17:49:13,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.3720 sec 2024-11-19T17:49:13,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/610a7b2a46734f16998b65c0ecd89d5c is 50, key is test_row_0/A:col10/1732038553830/Put/seqid=0 2024-11-19T17:49:13,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742091_1267 (size=14341) 2024-11-19T17:49:13,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/610a7b2a46734f16998b65c0ecd89d5c 2024-11-19T17:49:13,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/e5b9344cffa74745803b6954d7c667e8 is 50, key is test_row_0/B:col10/1732038553830/Put/seqid=0 2024-11-19T17:49:13,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038613910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038613913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:13,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742092_1268 (size=12001) 2024-11-19T17:49:13,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/e5b9344cffa74745803b6954d7c667e8 2024-11-19T17:49:13,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/5de7f14d056944c8914b2d91f1e1ebe8 is 50, key is test_row_0/C:col10/1732038553830/Put/seqid=0 2024-11-19T17:49:13,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742093_1269 (size=12001) 2024-11-19T17:49:13,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/5de7f14d056944c8914b2d91f1e1ebe8 2024-11-19T17:49:13,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/610a7b2a46734f16998b65c0ecd89d5c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/610a7b2a46734f16998b65c0ecd89d5c 2024-11-19T17:49:14,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/610a7b2a46734f16998b65c0ecd89d5c, entries=200, sequenceid=51, filesize=14.0 K 2024-11-19T17:49:14,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/e5b9344cffa74745803b6954d7c667e8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e5b9344cffa74745803b6954d7c667e8 2024-11-19T17:49:14,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e5b9344cffa74745803b6954d7c667e8, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T17:49:14,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/5de7f14d056944c8914b2d91f1e1ebe8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5de7f14d056944c8914b2d91f1e1ebe8 2024-11-19T17:49:14,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038614019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5de7f14d056944c8914b2d91f1e1ebe8, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T17:49:14,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f7aa8054d507e358fea7889f8e6a6f0a in 194ms, sequenceid=51, compaction requested=true 2024-11-19T17:49:14,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:14,028 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:14,028 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:14,029 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:14,029 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:14,029 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:14,029 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/59a66c219831455bbecd474cbde0817d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8b73216d25c54d5b84bf7bb9586e9e17, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e5b9344cffa74745803b6954d7c667e8] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.2 K 2024-11-19T17:49:14,029 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:14,029 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:14,029 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:14,030 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/49b52b0be6a0439395597a9e94fc0341, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2a97f695fbc44d9805d82b74327db3c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/610a7b2a46734f16998b65c0ecd89d5c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=39.7 K 2024-11-19T17:49:14,030 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a66c219831455bbecd474cbde0817d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732038552487 2024-11-19T17:49:14,030 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49b52b0be6a0439395597a9e94fc0341, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732038552487 2024-11-19T17:49:14,030 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b73216d25c54d5b84bf7bb9586e9e17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038552551 2024-11-19T17:49:14,031 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a97f695fbc44d9805d82b74327db3c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038552551 2024-11-19T17:49:14,031 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b9344cffa74745803b6954d7c667e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732038553712 2024-11-19T17:49:14,032 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 610a7b2a46734f16998b65c0ecd89d5c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732038553712 2024-11-19T17:49:14,032 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:14,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T17:49:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:14,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/669ab5429f934646adfb4a728a1344dd is 50, key is test_row_0/A:col10/1732038553908/Put/seqid=0 2024-11-19T17:49:14,051 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:14,051 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:14,051 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/e4b4f16382f0499d8e0df52b75e12077 is 50, key is test_row_0/B:col10/1732038553830/Put/seqid=0 2024-11-19T17:49:14,051 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/450a65029cc540e5a953a3f94eda69f7 is 50, key is test_row_0/A:col10/1732038553830/Put/seqid=0 2024-11-19T17:49:14,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038614052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038614055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038614056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742095_1271 (size=12104) 2024-11-19T17:49:14,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742094_1270 (size=12001) 2024-11-19T17:49:14,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742096_1272 (size=12104) 2024-11-19T17:49:14,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/669ab5429f934646adfb4a728a1344dd 2024-11-19T17:49:14,115 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/450a65029cc540e5a953a3f94eda69f7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/450a65029cc540e5a953a3f94eda69f7 2024-11-19T17:49:14,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/843f7b156694429eb65d76130c9e5ca5 is 50, key is test_row_0/B:col10/1732038553908/Put/seqid=0 2024-11-19T17:49:14,123 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into 450a65029cc540e5a953a3f94eda69f7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:14,123 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:14,123 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=13, startTime=1732038554028; duration=0sec 2024-11-19T17:49:14,123 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:14,123 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:14,123 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:14,125 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:14,125 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:14,125 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:14,126 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e09052e0239c46878990b63dc461cad7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39afe135b6924d74b4d7390ca8af1cd2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5de7f14d056944c8914b2d91f1e1ebe8] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.2 K 2024-11-19T17:49:14,127 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09052e0239c46878990b63dc461cad7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732038552487 2024-11-19T17:49:14,127 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39afe135b6924d74b4d7390ca8af1cd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038552551 2024-11-19T17:49:14,128 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5de7f14d056944c8914b2d91f1e1ebe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732038553712 2024-11-19T17:49:14,142 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:14,143 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/90f2f5527c4c40de9345ccf8c2d0b700 is 50, key is test_row_0/C:col10/1732038553830/Put/seqid=0 2024-11-19T17:49:14,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742098_1274 (size=12104) 2024-11-19T17:49:14,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742097_1273 (size=12001) 2024-11-19T17:49:14,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/843f7b156694429eb65d76130c9e5ca5 2024-11-19T17:49:14,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038614158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038614169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038614169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/17df65869cf24c87aa47b6ac03b9653a is 50, key is test_row_0/C:col10/1732038553908/Put/seqid=0 2024-11-19T17:49:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742099_1275 (size=12001) 2024-11-19T17:49:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038614228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038614368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038614379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038614379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,479 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/e4b4f16382f0499d8e0df52b75e12077 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e4b4f16382f0499d8e0df52b75e12077 2024-11-19T17:49:14,484 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into e4b4f16382f0499d8e0df52b75e12077(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:14,484 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:14,485 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=13, startTime=1732038554028; duration=0sec 2024-11-19T17:49:14,485 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:14,485 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038614538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,560 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/90f2f5527c4c40de9345ccf8c2d0b700 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/90f2f5527c4c40de9345ccf8c2d0b700 2024-11-19T17:49:14,565 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into 90f2f5527c4c40de9345ccf8c2d0b700(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:14,565 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:14,566 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=13, startTime=1732038554028; duration=0sec 2024-11-19T17:49:14,566 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:14,566 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:14,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/17df65869cf24c87aa47b6ac03b9653a 2024-11-19T17:49:14,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/669ab5429f934646adfb4a728a1344dd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/669ab5429f934646adfb4a728a1344dd 2024-11-19T17:49:14,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/669ab5429f934646adfb4a728a1344dd, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T17:49:14,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/843f7b156694429eb65d76130c9e5ca5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/843f7b156694429eb65d76130c9e5ca5 2024-11-19T17:49:14,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/843f7b156694429eb65d76130c9e5ca5, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T17:49:14,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/17df65869cf24c87aa47b6ac03b9653a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/17df65869cf24c87aa47b6ac03b9653a 2024-11-19T17:49:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:49:14,619 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:14,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T17:49:14,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/17df65869cf24c87aa47b6ac03b9653a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T17:49:14,623 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T17:49:14,624 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:14,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f7aa8054d507e358fea7889f8e6a6f0a in 592ms, sequenceid=77, compaction requested=false 2024-11-19T17:49:14,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:14,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:14,679 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:14,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:49:14,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/3e25b34fff4f44e68b4530c39f1747d0 is 50, key is test_row_0/A:col10/1732038554678/Put/seqid=0 2024-11-19T17:49:14,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742100_1276 (size=14341) 2024-11-19T17:49:14,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/3e25b34fff4f44e68b4530c39f1747d0 2024-11-19T17:49:14,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f53bd119b1084e4bbcd2b6e39c6785ed is 50, key is test_row_0/B:col10/1732038554678/Put/seqid=0 2024-11-19T17:49:14,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742101_1277 (size=12001) 2024-11-19T17:49:14,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038614724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T17:49:14,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038614724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038614728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038614729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T17:49:14,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:14,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038614839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038614839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038614839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038614839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:14,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T17:49:14,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:14,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:14,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:14,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:14,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T17:49:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038615049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038615049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038615049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038615049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038615049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T17:49:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:15,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:15,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f53bd119b1084e4bbcd2b6e39c6785ed 2024-11-19T17:49:15,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/5cce96f72fc54449b22917d5e440186f is 50, key is test_row_0/C:col10/1732038554678/Put/seqid=0 2024-11-19T17:49:15,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742102_1278 (size=12001) 2024-11-19T17:49:15,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/5cce96f72fc54449b22917d5e440186f 2024-11-19T17:49:15,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/3e25b34fff4f44e68b4530c39f1747d0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3e25b34fff4f44e68b4530c39f1747d0 2024-11-19T17:49:15,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3e25b34fff4f44e68b4530c39f1747d0, entries=200, sequenceid=91, filesize=14.0 K 2024-11-19T17:49:15,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f53bd119b1084e4bbcd2b6e39c6785ed as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f53bd119b1084e4bbcd2b6e39c6785ed 2024-11-19T17:49:15,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f53bd119b1084e4bbcd2b6e39c6785ed, entries=150, sequenceid=91, filesize=11.7 K 2024-11-19T17:49:15,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/5cce96f72fc54449b22917d5e440186f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5cce96f72fc54449b22917d5e440186f 2024-11-19T17:49:15,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5cce96f72fc54449b22917d5e440186f, entries=150, sequenceid=91, filesize=11.7 K 2024-11-19T17:49:15,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f7aa8054d507e358fea7889f8e6a6f0a in 506ms, sequenceid=91, compaction requested=true 2024-11-19T17:49:15,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:15,186 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:15,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:15,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:15,186 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:15,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:15,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:15,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:15,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:15,187 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:15,187 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:15,187 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:15,187 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/450a65029cc540e5a953a3f94eda69f7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/669ab5429f934646adfb4a728a1344dd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3e25b34fff4f44e68b4530c39f1747d0] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=37.5 K 2024-11-19T17:49:15,188 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:15,188 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:15,188 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:15,188 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e4b4f16382f0499d8e0df52b75e12077, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/843f7b156694429eb65d76130c9e5ca5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f53bd119b1084e4bbcd2b6e39c6785ed] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.3 K 2024-11-19T17:49:15,189 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 450a65029cc540e5a953a3f94eda69f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732038553712 2024-11-19T17:49:15,197 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 669ab5429f934646adfb4a728a1344dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732038553899 2024-11-19T17:49:15,197 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b4f16382f0499d8e0df52b75e12077, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732038553712 2024-11-19T17:49:15,197 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e25b34fff4f44e68b4530c39f1747d0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732038554043 2024-11-19T17:49:15,198 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 843f7b156694429eb65d76130c9e5ca5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732038553899 2024-11-19T17:49:15,198 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting f53bd119b1084e4bbcd2b6e39c6785ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732038554043 2024-11-19T17:49:15,221 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:15,222 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/7d52017d6c5b43e18b7f12e428570fb6 is 50, key is test_row_0/A:col10/1732038554678/Put/seqid=0 2024-11-19T17:49:15,227 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#232 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:15,227 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/0af9f6268db04272a83b286a09573644 is 50, key is test_row_0/B:col10/1732038554678/Put/seqid=0 2024-11-19T17:49:15,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T17:49:15,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:15,236 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:49:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T17:49:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/c878b2d098f94cc59fcff771db1113b8 is 50, key is test_row_0/A:col10/1732038554727/Put/seqid=0 2024-11-19T17:49:15,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742103_1279 (size=12207) 2024-11-19T17:49:15,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742104_1280 (size=12207) 2024-11-19T17:49:15,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742105_1281 (size=12001) 2024-11-19T17:49:15,296 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/c878b2d098f94cc59fcff771db1113b8 2024-11-19T17:49:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/c11e0b3f20ab4d28a0359f6c7cdcc34d is 50, key is test_row_0/B:col10/1732038554727/Put/seqid=0 2024-11-19T17:49:15,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742106_1282 (size=12001) 2024-11-19T17:49:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:15,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:15,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038615367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038615367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038615369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038615370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,425 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:49:15,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038615479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038615479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038615479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038615488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,674 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/7d52017d6c5b43e18b7f12e428570fb6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7d52017d6c5b43e18b7f12e428570fb6 2024-11-19T17:49:15,678 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into 7d52017d6c5b43e18b7f12e428570fb6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:15,678 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:15,678 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=13, startTime=1732038555186; duration=0sec 2024-11-19T17:49:15,678 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:15,678 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:15,678 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:15,679 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:15,679 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:15,679 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:15,679 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/90f2f5527c4c40de9345ccf8c2d0b700, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/17df65869cf24c87aa47b6ac03b9653a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5cce96f72fc54449b22917d5e440186f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.3 K 2024-11-19T17:49:15,680 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f2f5527c4c40de9345ccf8c2d0b700, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732038553712 2024-11-19T17:49:15,680 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17df65869cf24c87aa47b6ac03b9653a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732038553899 2024-11-19T17:49:15,680 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cce96f72fc54449b22917d5e440186f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732038554043 2024-11-19T17:49:15,689 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:15,690 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/e5b5440c40fc42fb8a01a3de19c8821d is 50, key is test_row_0/C:col10/1732038554678/Put/seqid=0 2024-11-19T17:49:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038615688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038615688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038615689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,696 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/0af9f6268db04272a83b286a09573644 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0af9f6268db04272a83b286a09573644 2024-11-19T17:49:15,701 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into 0af9f6268db04272a83b286a09573644(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:15,701 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:15,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:15,702 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=13, startTime=1732038555186; duration=0sec 2024-11-19T17:49:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038615698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:15,702 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:15,702 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:15,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742107_1283 (size=12207) 2024-11-19T17:49:15,715 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/c11e0b3f20ab4d28a0359f6c7cdcc34d 2024-11-19T17:49:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/9ddcdaba7c554e7eb4b3bcb3fdb50956 is 50, key is test_row_0/C:col10/1732038554727/Put/seqid=0 2024-11-19T17:49:15,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742108_1284 (size=12001) 2024-11-19T17:49:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T17:49:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038615999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038615999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038615999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038616008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038616058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,062 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:16,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,109 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/e5b5440c40fc42fb8a01a3de19c8821d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e5b5440c40fc42fb8a01a3de19c8821d 2024-11-19T17:49:16,120 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into e5b5440c40fc42fb8a01a3de19c8821d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:16,120 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:16,120 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=13, startTime=1732038555186; duration=0sec 2024-11-19T17:49:16,120 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:16,120 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:16,131 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/9ddcdaba7c554e7eb4b3bcb3fdb50956 2024-11-19T17:49:16,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/c878b2d098f94cc59fcff771db1113b8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c878b2d098f94cc59fcff771db1113b8 2024-11-19T17:49:16,145 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c878b2d098f94cc59fcff771db1113b8, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T17:49:16,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/c11e0b3f20ab4d28a0359f6c7cdcc34d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/c11e0b3f20ab4d28a0359f6c7cdcc34d 2024-11-19T17:49:16,151 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/c11e0b3f20ab4d28a0359f6c7cdcc34d, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T17:49:16,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/9ddcdaba7c554e7eb4b3bcb3fdb50956 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/9ddcdaba7c554e7eb4b3bcb3fdb50956 2024-11-19T17:49:16,156 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/9ddcdaba7c554e7eb4b3bcb3fdb50956, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T17:49:16,157 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f7aa8054d507e358fea7889f8e6a6f0a in 921ms, sequenceid=116, compaction requested=false 2024-11-19T17:49:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T17:49:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T17:49:16,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T17:49:16,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5350 sec 2024-11-19T17:49:16,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5420 sec 2024-11-19T17:49:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:16,510 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:16,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:49:16,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/ef09ff920ee847389d6df8e66c71a830 is 50, key is test_row_0/A:col10/1732038555369/Put/seqid=0 2024-11-19T17:49:16,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742109_1285 (size=12051) 2024-11-19T17:49:16,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038616544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038616545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038616545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038616548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038616659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038616659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038616659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038616659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T17:49:16,779 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:16,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T17:49:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T17:49:16,784 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:16,784 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:16,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:16,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038616869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038616869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038616869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038616869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T17:49:16,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/ef09ff920ee847389d6df8e66c71a830 2024-11-19T17:49:16,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/55e6f67418c04704ad1c9cfe077b0bc5 is 50, key is test_row_0/B:col10/1732038555369/Put/seqid=0 2024-11-19T17:49:16,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T17:49:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:16,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:16,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742110_1286 (size=12051) 2024-11-19T17:49:16,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/55e6f67418c04704ad1c9cfe077b0bc5 2024-11-19T17:49:16,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/ecdee2e805a848c5a1046612dc9a8815 is 50, key is test_row_0/C:col10/1732038555369/Put/seqid=0 2024-11-19T17:49:17,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742111_1287 (size=12051) 2024-11-19T17:49:17,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/ecdee2e805a848c5a1046612dc9a8815 2024-11-19T17:49:17,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/ef09ff920ee847389d6df8e66c71a830 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ef09ff920ee847389d6df8e66c71a830 2024-11-19T17:49:17,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ef09ff920ee847389d6df8e66c71a830, entries=150, sequenceid=132, filesize=11.8 K 2024-11-19T17:49:17,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/55e6f67418c04704ad1c9cfe077b0bc5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/55e6f67418c04704ad1c9cfe077b0bc5 2024-11-19T17:49:17,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/55e6f67418c04704ad1c9cfe077b0bc5, entries=150, sequenceid=132, filesize=11.8 K 2024-11-19T17:49:17,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/ecdee2e805a848c5a1046612dc9a8815 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ecdee2e805a848c5a1046612dc9a8815 2024-11-19T17:49:17,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ecdee2e805a848c5a1046612dc9a8815, entries=150, sequenceid=132, filesize=11.8 K 2024-11-19T17:49:17,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7aa8054d507e358fea7889f8e6a6f0a in 529ms, sequenceid=132, compaction requested=true 2024-11-19T17:49:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:17,039 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:17,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:17,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:17,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:17,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:49:17,040 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:17,040 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:17,041 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:17,041 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:17,041 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7d52017d6c5b43e18b7f12e428570fb6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c878b2d098f94cc59fcff771db1113b8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ef09ff920ee847389d6df8e66c71a830] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.4 K 2024-11-19T17:49:17,041 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:17,041 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:17,041 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:17,041 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0af9f6268db04272a83b286a09573644, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/c11e0b3f20ab4d28a0359f6c7cdcc34d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/55e6f67418c04704ad1c9cfe077b0bc5] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.4 K 2024-11-19T17:49:17,041 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d52017d6c5b43e18b7f12e428570fb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732038554043 2024-11-19T17:49:17,042 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af9f6268db04272a83b286a09573644, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732038554043 2024-11-19T17:49:17,042 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c878b2d098f94cc59fcff771db1113b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732038554723 2024-11-19T17:49:17,042 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c11e0b3f20ab4d28a0359f6c7cdcc34d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732038554723 2024-11-19T17:49:17,042 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 55e6f67418c04704ad1c9cfe077b0bc5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732038555365 2024-11-19T17:49:17,043 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef09ff920ee847389d6df8e66c71a830, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732038555365 2024-11-19T17:49:17,054 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:17,055 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/5f02d679673a4fe29f4e72ea6456106d is 50, key is test_row_0/A:col10/1732038555369/Put/seqid=0 2024-11-19T17:49:17,062 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:17,063 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/a6c627a5a80c46bdb276012d3c686b7a is 50, key is test_row_0/B:col10/1732038555369/Put/seqid=0 2024-11-19T17:49:17,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742113_1289 (size=12359) 2024-11-19T17:49:17,092 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/a6c627a5a80c46bdb276012d3c686b7a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/a6c627a5a80c46bdb276012d3c686b7a 2024-11-19T17:49:17,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T17:49:17,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:17,094 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T17:49:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T17:49:17,101 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into a6c627a5a80c46bdb276012d3c686b7a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:17,101 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:17,101 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=13, startTime=1732038557039; duration=0sec 2024-11-19T17:49:17,102 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:17,102 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:17,102 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:17,105 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:17,105 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:17,105 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:17,106 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e5b5440c40fc42fb8a01a3de19c8821d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/9ddcdaba7c554e7eb4b3bcb3fdb50956, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ecdee2e805a848c5a1046612dc9a8815] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.4 K 2024-11-19T17:49:17,106 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b5440c40fc42fb8a01a3de19c8821d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732038554043 2024-11-19T17:49:17,106 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ddcdaba7c554e7eb4b3bcb3fdb50956, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732038554723 2024-11-19T17:49:17,107 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ecdee2e805a848c5a1046612dc9a8815, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732038555365 2024-11-19T17:49:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/ae1d928d274d42898ce77efc6d36e6ae is 50, key is test_row_0/A:col10/1732038556544/Put/seqid=0 2024-11-19T17:49:17,121 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:17,122 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/c63fa0417dea41bc8b372b670ef1d40e is 50, key is test_row_0/C:col10/1732038555369/Put/seqid=0 2024-11-19T17:49:17,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742112_1288 (size=12359) 2024-11-19T17:49:17,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742114_1290 (size=12151) 2024-11-19T17:49:17,136 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/ae1d928d274d42898ce77efc6d36e6ae 2024-11-19T17:49:17,138 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/5f02d679673a4fe29f4e72ea6456106d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f02d679673a4fe29f4e72ea6456106d 2024-11-19T17:49:17,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into 5f02d679673a4fe29f4e72ea6456106d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:17,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:17,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=13, startTime=1732038557039; duration=0sec 2024-11-19T17:49:17,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:17,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:17,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742115_1291 (size=12359) 2024-11-19T17:49:17,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/6e171de3268a430eb43b68acb828e0cd is 50, key is test_row_0/B:col10/1732038556544/Put/seqid=0 2024-11-19T17:49:17,160 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/c63fa0417dea41bc8b372b670ef1d40e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c63fa0417dea41bc8b372b670ef1d40e 2024-11-19T17:49:17,171 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into c63fa0417dea41bc8b372b670ef1d40e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:17,171 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:17,171 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=13, startTime=1732038557040; duration=0sec 2024-11-19T17:49:17,171 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:17,171 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:17,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742116_1292 (size=12151) 2024-11-19T17:49:17,180 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/6e171de3268a430eb43b68acb828e0cd 2024-11-19T17:49:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:17,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:17,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/8be100ca36a3408b8259130d85fae6e4 is 50, key is test_row_0/C:col10/1732038556544/Put/seqid=0 2024-11-19T17:49:17,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038617193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038617194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038617198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038617198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742117_1293 (size=12151) 2024-11-19T17:49:17,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038617308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038617308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038617309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038617309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T17:49:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038617519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038617519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038617519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038617519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,666 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/8be100ca36a3408b8259130d85fae6e4 2024-11-19T17:49:17,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/ae1d928d274d42898ce77efc6d36e6ae as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ae1d928d274d42898ce77efc6d36e6ae 2024-11-19T17:49:17,673 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ae1d928d274d42898ce77efc6d36e6ae, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T17:49:17,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/6e171de3268a430eb43b68acb828e0cd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6e171de3268a430eb43b68acb828e0cd 2024-11-19T17:49:17,677 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6e171de3268a430eb43b68acb828e0cd, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T17:49:17,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/8be100ca36a3408b8259130d85fae6e4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8be100ca36a3408b8259130d85fae6e4 2024-11-19T17:49:17,682 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8be100ca36a3408b8259130d85fae6e4, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T17:49:17,683 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f7aa8054d507e358fea7889f8e6a6f0a in 590ms, sequenceid=155, compaction requested=false 2024-11-19T17:49:17,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:17,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:17,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T17:49:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T17:49:17,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T17:49:17,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 900 msec 2024-11-19T17:49:17,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 908 msec 2024-11-19T17:49:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:17,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:17,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T17:49:17,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/1f62b2fc7839441fae99f41bf0ca30c3 is 50, key is test_row_0/A:col10/1732038557831/Put/seqid=0 2024-11-19T17:49:17,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742118_1294 (size=12151) 2024-11-19T17:49:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038617851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038617852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038617853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038617856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T17:49:17,919 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:17,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T17:49:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T17:49:17,921 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:17,922 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:17,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:17,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038617959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038617969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038617969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038617969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T17:49:18,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038618068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,071 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T17:49:18,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:18,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038618169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038618178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038618178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038618178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T17:49:18,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:18,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T17:49:18,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/1f62b2fc7839441fae99f41bf0ca30c3 2024-11-19T17:49:18,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f5bfb7bbc97445a39a9b8dc06e21e60e is 50, key is test_row_0/B:col10/1732038557831/Put/seqid=0 2024-11-19T17:49:18,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742119_1295 (size=12151) 2024-11-19T17:49:18,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T17:49:18,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:18,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038618479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038618488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038618488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038618488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T17:49:18,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:18,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T17:49:18,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f5bfb7bbc97445a39a9b8dc06e21e60e 2024-11-19T17:49:18,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T17:49:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:18,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/39a674b1950147b38bbf4ddd67a8c267 is 50, key is test_row_0/C:col10/1732038557831/Put/seqid=0 2024-11-19T17:49:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742120_1296 (size=12151) 2024-11-19T17:49:18,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T17:49:18,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:18,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038618988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:18,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T17:49:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:18,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:19,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038618998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:19,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:19,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038618999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038618999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T17:49:19,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/39a674b1950147b38bbf4ddd67a8c267 2024-11-19T17:49:19,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/1f62b2fc7839441fae99f41bf0ca30c3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1f62b2fc7839441fae99f41bf0ca30c3 2024-11-19T17:49:19,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1f62b2fc7839441fae99f41bf0ca30c3, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T17:49:19,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f5bfb7bbc97445a39a9b8dc06e21e60e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f5bfb7bbc97445a39a9b8dc06e21e60e 2024-11-19T17:49:19,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f5bfb7bbc97445a39a9b8dc06e21e60e, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T17:49:19,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/39a674b1950147b38bbf4ddd67a8c267 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39a674b1950147b38bbf4ddd67a8c267 2024-11-19T17:49:19,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39a674b1950147b38bbf4ddd67a8c267, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T17:49:19,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f7aa8054d507e358fea7889f8e6a6f0a in 1281ms, sequenceid=173, compaction requested=true 2024-11-19T17:49:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:19,111 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:19,111 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:19,112 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:19,112 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:19,112 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:19,112 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:19,112 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:19,112 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:19,112 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/a6c627a5a80c46bdb276012d3c686b7a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6e171de3268a430eb43b68acb828e0cd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f5bfb7bbc97445a39a9b8dc06e21e60e] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.8 K 2024-11-19T17:49:19,112 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f02d679673a4fe29f4e72ea6456106d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ae1d928d274d42898ce77efc6d36e6ae, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1f62b2fc7839441fae99f41bf0ca30c3] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.8 K 2024-11-19T17:49:19,112 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c627a5a80c46bdb276012d3c686b7a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732038555365 2024-11-19T17:49:19,112 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f02d679673a4fe29f4e72ea6456106d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732038555365 2024-11-19T17:49:19,112 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae1d928d274d42898ce77efc6d36e6ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038556536 2024-11-19T17:49:19,112 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e171de3268a430eb43b68acb828e0cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038556536 2024-11-19T17:49:19,113 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f62b2fc7839441fae99f41bf0ca30c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732038557191 2024-11-19T17:49:19,114 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting f5bfb7bbc97445a39a9b8dc06e21e60e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732038557191 2024-11-19T17:49:19,119 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:19,119 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/d650433b26a04572867c7517c35cc980 is 50, key is test_row_0/A:col10/1732038557831/Put/seqid=0 2024-11-19T17:49:19,129 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:19,129 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f7ad8b1553bd4fee8a5925bc8a12fc9d is 50, key is test_row_0/B:col10/1732038557831/Put/seqid=0 2024-11-19T17:49:19,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742121_1297 (size=12561) 2024-11-19T17:49:19,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742122_1298 (size=12561) 2024-11-19T17:49:19,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T17:49:19,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:19,148 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T17:49:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/97a78895520944a5812fc6fa550f5eec is 50, key is test_row_0/A:col10/1732038557851/Put/seqid=0 2024-11-19T17:49:19,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742123_1299 (size=12151) 2024-11-19T17:49:19,539 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/d650433b26a04572867c7517c35cc980 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d650433b26a04572867c7517c35cc980 2024-11-19T17:49:19,539 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f7ad8b1553bd4fee8a5925bc8a12fc9d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f7ad8b1553bd4fee8a5925bc8a12fc9d 2024-11-19T17:49:19,547 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into d650433b26a04572867c7517c35cc980(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:19,547 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into f7ad8b1553bd4fee8a5925bc8a12fc9d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:19,548 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:19,548 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:19,548 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=13, startTime=1732038559111; duration=0sec 2024-11-19T17:49:19,548 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=13, startTime=1732038559111; duration=0sec 2024-11-19T17:49:19,548 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:19,548 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:19,548 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:19,548 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:19,548 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:19,549 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:19,550 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:19,550 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:19,550 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c63fa0417dea41bc8b372b670ef1d40e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8be100ca36a3408b8259130d85fae6e4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39a674b1950147b38bbf4ddd67a8c267] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=35.8 K 2024-11-19T17:49:19,550 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c63fa0417dea41bc8b372b670ef1d40e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732038555365 2024-11-19T17:49:19,550 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be100ca36a3408b8259130d85fae6e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038556536 2024-11-19T17:49:19,551 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a674b1950147b38bbf4ddd67a8c267, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732038557191 2024-11-19T17:49:19,556 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/97a78895520944a5812fc6fa550f5eec 2024-11-19T17:49:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/53807495ba4d4412b41aed846cd55f88 is 50, key is test_row_0/B:col10/1732038557851/Put/seqid=0 2024-11-19T17:49:19,569 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#252 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:19,569 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/388d5e0107cc4641aa7790509c11b1d1 is 50, key is test_row_0/C:col10/1732038557831/Put/seqid=0 2024-11-19T17:49:19,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742124_1300 (size=12151) 2024-11-19T17:49:19,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742125_1301 (size=12561) 2024-11-19T17:49:19,600 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/388d5e0107cc4641aa7790509c11b1d1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/388d5e0107cc4641aa7790509c11b1d1 2024-11-19T17:49:19,606 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into 388d5e0107cc4641aa7790509c11b1d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:19,606 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:19,606 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=13, startTime=1732038559111; duration=0sec 2024-11-19T17:49:19,606 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:19,606 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:19,983 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/53807495ba4d4412b41aed846cd55f88 2024-11-19T17:49:19,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/ef61b1f74853460980d8c13f9a680ccb is 50, key is test_row_0/C:col10/1732038557851/Put/seqid=0 2024-11-19T17:49:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:20,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:20,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742126_1302 (size=12151) 2024-11-19T17:49:20,010 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/ef61b1f74853460980d8c13f9a680ccb 2024-11-19T17:49:20,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/97a78895520944a5812fc6fa550f5eec as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/97a78895520944a5812fc6fa550f5eec 2024-11-19T17:49:20,018 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/97a78895520944a5812fc6fa550f5eec, entries=150, sequenceid=194, filesize=11.9 K 2024-11-19T17:49:20,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/53807495ba4d4412b41aed846cd55f88 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/53807495ba4d4412b41aed846cd55f88 2024-11-19T17:49:20,023 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/53807495ba4d4412b41aed846cd55f88, entries=150, sequenceid=194, filesize=11.9 K 2024-11-19T17:49:20,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/ef61b1f74853460980d8c13f9a680ccb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ef61b1f74853460980d8c13f9a680ccb 2024-11-19T17:49:20,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038620021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,028 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ef61b1f74853460980d8c13f9a680ccb, entries=150, sequenceid=194, filesize=11.9 K 2024-11-19T17:49:20,029 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for f7aa8054d507e358fea7889f8e6a6f0a in 881ms, sequenceid=194, compaction requested=false 2024-11-19T17:49:20,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:20,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T17:49:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T17:49:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:20,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:20,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T17:49:20,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/b67450a4a9bb46a387bdacd5a9a4d60f is 50, key is test_row_0/A:col10/1732038560030/Put/seqid=0 2024-11-19T17:49:20,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T17:49:20,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1080 sec 2024-11-19T17:49:20,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1170 sec 2024-11-19T17:49:20,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742127_1303 (size=14541) 2024-11-19T17:49:20,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/b67450a4a9bb46a387bdacd5a9a4d60f 2024-11-19T17:49:20,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/47546fa1da7941dfb12485a33a4dbc1e is 50, key is test_row_0/B:col10/1732038560030/Put/seqid=0 2024-11-19T17:49:20,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742128_1304 (size=12151) 2024-11-19T17:49:20,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038620055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038620058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038620061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T17:49:20,069 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:20,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T17:49:20,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:49:20,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:20,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038620128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038620168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038620168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:49:20,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038620188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:49:20,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:20,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038620338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:49:20,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:20,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038620378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038620379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:49:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038620398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/47546fa1da7941dfb12485a33a4dbc1e 2024-11-19T17:49:20,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/32035be042864b5c9250f37e9e4a5f4d is 50, key is test_row_0/C:col10/1732038560030/Put/seqid=0 2024-11-19T17:49:20,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742129_1305 (size=12151) 2024-11-19T17:49:20,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:49:20,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:20,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038620649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:49:20,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:20,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038620688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038620688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:49:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038620709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:20,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:49:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:20,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/32035be042864b5c9250f37e9e4a5f4d 2024-11-19T17:49:20,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/b67450a4a9bb46a387bdacd5a9a4d60f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b67450a4a9bb46a387bdacd5a9a4d60f 2024-11-19T17:49:20,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b67450a4a9bb46a387bdacd5a9a4d60f, entries=200, sequenceid=214, filesize=14.2 K 2024-11-19T17:49:20,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/47546fa1da7941dfb12485a33a4dbc1e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/47546fa1da7941dfb12485a33a4dbc1e 2024-11-19T17:49:20,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/47546fa1da7941dfb12485a33a4dbc1e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T17:49:20,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/32035be042864b5c9250f37e9e4a5f4d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/32035be042864b5c9250f37e9e4a5f4d 2024-11-19T17:49:20,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/32035be042864b5c9250f37e9e4a5f4d, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T17:49:20,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f7aa8054d507e358fea7889f8e6a6f0a in 850ms, sequenceid=214, compaction requested=true 2024-11-19T17:49:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:20,881 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:20,881 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:20,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:20,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:20,882 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:20,882 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,882 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:20,882 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,882 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d650433b26a04572867c7517c35cc980, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/97a78895520944a5812fc6fa550f5eec, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b67450a4a9bb46a387bdacd5a9a4d60f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=38.3 K 2024-11-19T17:49:20,882 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f7ad8b1553bd4fee8a5925bc8a12fc9d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/53807495ba4d4412b41aed846cd55f88, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/47546fa1da7941dfb12485a33a4dbc1e] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=36.0 K 2024-11-19T17:49:20,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting d650433b26a04572867c7517c35cc980, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732038557191 2024-11-19T17:49:20,882 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ad8b1553bd4fee8a5925bc8a12fc9d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732038557191 2024-11-19T17:49:20,883 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97a78895520944a5812fc6fa550f5eec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732038557837 2024-11-19T17:49:20,883 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 53807495ba4d4412b41aed846cd55f88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732038557837 2024-11-19T17:49:20,883 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b67450a4a9bb46a387bdacd5a9a4d60f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038560019 2024-11-19T17:49:20,883 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 47546fa1da7941dfb12485a33a4dbc1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038560024 2024-11-19T17:49:20,890 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:20,890 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:20,891 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/26b06eda72394745a3274dd0ea865440 is 50, key is test_row_0/B:col10/1732038560030/Put/seqid=0 2024-11-19T17:49:20,891 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/2a6b066f82da476db54db87bdfb0e969 is 50, key is test_row_0/A:col10/1732038560030/Put/seqid=0 2024-11-19T17:49:20,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742131_1307 (size=12663) 2024-11-19T17:49:20,908 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/2a6b066f82da476db54db87bdfb0e969 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/2a6b066f82da476db54db87bdfb0e969 2024-11-19T17:49:20,917 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into 2a6b066f82da476db54db87bdfb0e969(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:20,917 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:20,917 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=13, startTime=1732038560881; duration=0sec 2024-11-19T17:49:20,917 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:20,917 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:20,917 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:20,919 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:20,919 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:20,919 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,919 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/388d5e0107cc4641aa7790509c11b1d1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ef61b1f74853460980d8c13f9a680ccb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/32035be042864b5c9250f37e9e4a5f4d] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=36.0 K 2024-11-19T17:49:20,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742130_1306 (size=12663) 2024-11-19T17:49:20,920 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 388d5e0107cc4641aa7790509c11b1d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732038557191 2024-11-19T17:49:20,920 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef61b1f74853460980d8c13f9a680ccb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732038557837 2024-11-19T17:49:20,920 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32035be042864b5c9250f37e9e4a5f4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038560024 2024-11-19T17:49:20,930 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:20,931 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/4976eeb263684cfaa860133be160ef55 is 50, key is test_row_0/C:col10/1732038560030/Put/seqid=0 2024-11-19T17:49:20,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742132_1308 (size=12663) 2024-11-19T17:49:20,975 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:49:20,975 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T17:49:20,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:49:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:20,989 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T17:49:20,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/c6a648480ee9486892ef5491f275ee77 is 50, key is test_row_0/A:col10/1732038560047/Put/seqid=0 2024-11-19T17:49:20,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742133_1309 (size=12151) 2024-11-19T17:49:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:49:21,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038621209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038621211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038621213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038621219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038621318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038621318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,324 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/26b06eda72394745a3274dd0ea865440 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/26b06eda72394745a3274dd0ea865440 2024-11-19T17:49:21,328 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into 26b06eda72394745a3274dd0ea865440(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:21,328 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:21,328 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=13, startTime=1732038560881; duration=0sec 2024-11-19T17:49:21,328 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:21,328 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038621329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,342 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/4976eeb263684cfaa860133be160ef55 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/4976eeb263684cfaa860133be160ef55 2024-11-19T17:49:21,345 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into 4976eeb263684cfaa860133be160ef55(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:21,346 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:21,346 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=13, startTime=1732038560881; duration=0sec 2024-11-19T17:49:21,346 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:21,346 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:21,399 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/c6a648480ee9486892ef5491f275ee77 2024-11-19T17:49:21,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/34dba89f6eb54f9c96e3de1027c6de24 is 50, key is test_row_0/B:col10/1732038560047/Put/seqid=0 2024-11-19T17:49:21,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742134_1310 (size=12151) 2024-11-19T17:49:21,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038621528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038621528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038621539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,809 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/34dba89f6eb54f9c96e3de1027c6de24 2024-11-19T17:49:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/6fddb325d23949bda2ec503fbc307fc2 is 50, key is test_row_0/C:col10/1732038560047/Put/seqid=0 2024-11-19T17:49:21,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742135_1311 (size=12151) 2024-11-19T17:49:21,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038621838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038621838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:21,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038621848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038622078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,082 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,228 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/6fddb325d23949bda2ec503fbc307fc2 2024-11-19T17:49:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:49:22,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038622229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/c6a648480ee9486892ef5491f275ee77 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c6a648480ee9486892ef5491f275ee77 2024-11-19T17:49:22,232 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:22,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,235 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c6a648480ee9486892ef5491f275ee77, entries=150, sequenceid=234, filesize=11.9 K 2024-11-19T17:49:22,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/34dba89f6eb54f9c96e3de1027c6de24 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/34dba89f6eb54f9c96e3de1027c6de24 2024-11-19T17:49:22,239 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/34dba89f6eb54f9c96e3de1027c6de24, entries=150, sequenceid=234, filesize=11.9 K 2024-11-19T17:49:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/6fddb325d23949bda2ec503fbc307fc2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/6fddb325d23949bda2ec503fbc307fc2 2024-11-19T17:49:22,248 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/6fddb325d23949bda2ec503fbc307fc2, entries=150, sequenceid=234, filesize=11.9 K 2024-11-19T17:49:22,249 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for f7aa8054d507e358fea7889f8e6a6f0a in 1259ms, sequenceid=234, compaction requested=false 2024-11-19T17:49:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T17:49:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T17:49:22,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T17:49:22,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1770 sec 2024-11-19T17:49:22,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1840 sec 2024-11-19T17:49:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:22,351 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:22,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T17:49:22,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/3cdf59437c69422eab8bb87200c0af8f is 50, key is test_row_0/A:col10/1732038561197/Put/seqid=0 2024-11-19T17:49:22,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742136_1312 (size=16931) 2024-11-19T17:49:22,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038622379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038622382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038622383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038622489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038622490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038622490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038622699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038622699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038622699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:22,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/3cdf59437c69422eab8bb87200c0af8f 2024-11-19T17:49:22,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/cda466ad4bd34469a41eef3cce379676 is 50, key is test_row_0/B:col10/1732038561197/Put/seqid=0 2024-11-19T17:49:22,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742137_1313 (size=12151) 2024-11-19T17:49:23,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038623008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038623009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038623009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/cda466ad4bd34469a41eef3cce379676 2024-11-19T17:49:23,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/daa9af6937aa432ba431727e1bebdc8f is 50, key is test_row_0/C:col10/1732038561197/Put/seqid=0 2024-11-19T17:49:23,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742138_1314 (size=12151) 2024-11-19T17:49:23,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/daa9af6937aa432ba431727e1bebdc8f 2024-11-19T17:49:23,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/3cdf59437c69422eab8bb87200c0af8f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3cdf59437c69422eab8bb87200c0af8f 2024-11-19T17:49:23,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3cdf59437c69422eab8bb87200c0af8f, entries=250, sequenceid=255, filesize=16.5 K 2024-11-19T17:49:23,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/cda466ad4bd34469a41eef3cce379676 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/cda466ad4bd34469a41eef3cce379676 2024-11-19T17:49:23,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/cda466ad4bd34469a41eef3cce379676, entries=150, sequenceid=255, filesize=11.9 K 2024-11-19T17:49:23,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/daa9af6937aa432ba431727e1bebdc8f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/daa9af6937aa432ba431727e1bebdc8f 2024-11-19T17:49:23,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/daa9af6937aa432ba431727e1bebdc8f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-19T17:49:23,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for f7aa8054d507e358fea7889f8e6a6f0a in 886ms, sequenceid=255, compaction requested=true 2024-11-19T17:49:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:23,237 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:23,237 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:23,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:23,238 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:23,238 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:23,238 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:23,238 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:23,238 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:23,238 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:23,238 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/26b06eda72394745a3274dd0ea865440, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/34dba89f6eb54f9c96e3de1027c6de24, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/cda466ad4bd34469a41eef3cce379676] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=36.1 K 2024-11-19T17:49:23,238 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/2a6b066f82da476db54db87bdfb0e969, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c6a648480ee9486892ef5491f275ee77, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3cdf59437c69422eab8bb87200c0af8f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=40.8 K 2024-11-19T17:49:23,239 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a6b066f82da476db54db87bdfb0e969, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038560024 2024-11-19T17:49:23,239 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b06eda72394745a3274dd0ea865440, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038560024 2024-11-19T17:49:23,239 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a648480ee9486892ef5491f275ee77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732038560047 2024-11-19T17:49:23,239 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 34dba89f6eb54f9c96e3de1027c6de24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732038560047 2024-11-19T17:49:23,239 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cdf59437c69422eab8bb87200c0af8f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732038561179 2024-11-19T17:49:23,239 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting cda466ad4bd34469a41eef3cce379676, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732038561197 2024-11-19T17:49:23,249 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:23,249 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/999b4ed0d13d420985d13dc57aaa266e is 50, key is test_row_0/B:col10/1732038561197/Put/seqid=0 2024-11-19T17:49:23,249 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:23,250 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/a9935fd124be4edc99cb2a37bcb766c9 is 50, key is test_row_0/A:col10/1732038561197/Put/seqid=0 2024-11-19T17:49:23,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742139_1315 (size=12765) 2024-11-19T17:49:23,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742140_1316 (size=12765) 2024-11-19T17:49:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:23,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T17:49:23,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/b1e56598f5754f3abf2465d8b521b571 is 50, key is test_row_0/A:col10/1732038563519/Put/seqid=0 2024-11-19T17:49:23,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742141_1317 (size=12301) 2024-11-19T17:49:23,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038623545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038623545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038623550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038623658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038623659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038623659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,672 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/999b4ed0d13d420985d13dc57aaa266e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/999b4ed0d13d420985d13dc57aaa266e 2024-11-19T17:49:23,676 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into 999b4ed0d13d420985d13dc57aaa266e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:23,676 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:23,676 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=13, startTime=1732038563237; duration=0sec 2024-11-19T17:49:23,676 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:23,676 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:23,676 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:23,678 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:23,678 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:23,678 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:23,678 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/a9935fd124be4edc99cb2a37bcb766c9 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/a9935fd124be4edc99cb2a37bcb766c9 2024-11-19T17:49:23,678 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/4976eeb263684cfaa860133be160ef55, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/6fddb325d23949bda2ec503fbc307fc2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/daa9af6937aa432ba431727e1bebdc8f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=36.1 K 2024-11-19T17:49:23,678 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 4976eeb263684cfaa860133be160ef55, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732038560024 2024-11-19T17:49:23,678 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fddb325d23949bda2ec503fbc307fc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732038560047 2024-11-19T17:49:23,679 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting daa9af6937aa432ba431727e1bebdc8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732038561197 2024-11-19T17:49:23,682 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into a9935fd124be4edc99cb2a37bcb766c9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:23,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:23,682 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=13, startTime=1732038563237; duration=0sec 2024-11-19T17:49:23,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:23,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:23,685 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:23,685 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/0c8e54ffcc2144e491c21653d20b8f7a is 50, key is test_row_0/C:col10/1732038561197/Put/seqid=0 2024-11-19T17:49:23,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742142_1318 (size=12765) 2024-11-19T17:49:23,693 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/0c8e54ffcc2144e491c21653d20b8f7a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0c8e54ffcc2144e491c21653d20b8f7a 2024-11-19T17:49:23,697 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into 0c8e54ffcc2144e491c21653d20b8f7a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:23,697 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:23,697 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=13, startTime=1732038563237; duration=0sec 2024-11-19T17:49:23,697 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:23,697 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:23,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038623869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038623869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038623869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:23,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/b1e56598f5754f3abf2465d8b521b571 2024-11-19T17:49:23,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/7b3b06783f7349bc9b45f288ecd2c3d6 is 50, key is test_row_0/B:col10/1732038563519/Put/seqid=0 2024-11-19T17:49:23,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742143_1319 (size=12301) 2024-11-19T17:49:23,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/7b3b06783f7349bc9b45f288ecd2c3d6 2024-11-19T17:49:23,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/f88fa40958e343c7a95b6c7b57531f2c is 50, key is test_row_0/C:col10/1732038563519/Put/seqid=0 2024-11-19T17:49:24,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742144_1320 (size=12301) 2024-11-19T17:49:24,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/f88fa40958e343c7a95b6c7b57531f2c 2024-11-19T17:49:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/b1e56598f5754f3abf2465d8b521b571 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b1e56598f5754f3abf2465d8b521b571 2024-11-19T17:49:24,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b1e56598f5754f3abf2465d8b521b571, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T17:49:24,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/7b3b06783f7349bc9b45f288ecd2c3d6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/7b3b06783f7349bc9b45f288ecd2c3d6 2024-11-19T17:49:24,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/7b3b06783f7349bc9b45f288ecd2c3d6, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T17:49:24,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/f88fa40958e343c7a95b6c7b57531f2c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/f88fa40958e343c7a95b6c7b57531f2c 2024-11-19T17:49:24,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/f88fa40958e343c7a95b6c7b57531f2c, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T17:49:24,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for f7aa8054d507e358fea7889f8e6a6f0a in 533ms, sequenceid=274, compaction requested=false 2024-11-19T17:49:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:24,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:24,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T17:49:24,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/95260de172024e948db545b91b0235ae is 50, key is test_row_0/A:col10/1732038564180/Put/seqid=0 2024-11-19T17:49:24,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742145_1321 (size=14741) 2024-11-19T17:49:24,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038624206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038624208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038624212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:49:24,239 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:24,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T17:49:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:49:24,241 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:24,242 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:24,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038624258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,260 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038624319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038624319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038624319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:49:24,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T17:49:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:24,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038624529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038624530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038624530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T17:49:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:24,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:49:24,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/95260de172024e948db545b91b0235ae 2024-11-19T17:49:24,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/ba981e1a7b494cecbff5fdcf569e379d is 50, key is test_row_0/B:col10/1732038564180/Put/seqid=0 2024-11-19T17:49:24,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742146_1322 (size=12301) 2024-11-19T17:49:24,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/ba981e1a7b494cecbff5fdcf569e379d 2024-11-19T17:49:24,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/85121ff20bbf46a2aa80eddfb6990986 is 50, key is test_row_0/C:col10/1732038564180/Put/seqid=0 2024-11-19T17:49:24,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742147_1323 (size=12301) 2024-11-19T17:49:24,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T17:49:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:24,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038624839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038624839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038624848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:24,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T17:49:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:24,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:49:25,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T17:49:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:25,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:25,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/85121ff20bbf46a2aa80eddfb6990986 2024-11-19T17:49:25,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/95260de172024e948db545b91b0235ae as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/95260de172024e948db545b91b0235ae 2024-11-19T17:49:25,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/95260de172024e948db545b91b0235ae, entries=200, sequenceid=297, filesize=14.4 K 2024-11-19T17:49:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/ba981e1a7b494cecbff5fdcf569e379d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/ba981e1a7b494cecbff5fdcf569e379d 2024-11-19T17:49:25,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/ba981e1a7b494cecbff5fdcf569e379d, entries=150, sequenceid=297, filesize=12.0 K 2024-11-19T17:49:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/85121ff20bbf46a2aa80eddfb6990986 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/85121ff20bbf46a2aa80eddfb6990986 2024-11-19T17:49:25,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/85121ff20bbf46a2aa80eddfb6990986, entries=150, sequenceid=297, filesize=12.0 K 2024-11-19T17:49:25,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f7aa8054d507e358fea7889f8e6a6f0a in 947ms, sequenceid=297, compaction requested=true 2024-11-19T17:49:25,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:25,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:25,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:25,130 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:25,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:25,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:25,130 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:25,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:25,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:25,131 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:25,131 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:25,131 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:25,131 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/999b4ed0d13d420985d13dc57aaa266e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/7b3b06783f7349bc9b45f288ecd2c3d6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/ba981e1a7b494cecbff5fdcf569e379d] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=36.5 K 2024-11-19T17:49:25,131 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:25,131 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:25,131 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:25,131 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/a9935fd124be4edc99cb2a37bcb766c9, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b1e56598f5754f3abf2465d8b521b571, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/95260de172024e948db545b91b0235ae] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=38.9 K 2024-11-19T17:49:25,132 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9935fd124be4edc99cb2a37bcb766c9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732038561197 2024-11-19T17:49:25,132 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 999b4ed0d13d420985d13dc57aaa266e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732038561197 2024-11-19T17:49:25,132 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1e56598f5754f3abf2465d8b521b571, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732038562368 2024-11-19T17:49:25,132 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95260de172024e948db545b91b0235ae, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732038563549 2024-11-19T17:49:25,132 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b3b06783f7349bc9b45f288ecd2c3d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732038562368 2024-11-19T17:49:25,133 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ba981e1a7b494cecbff5fdcf569e379d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732038563549 2024-11-19T17:49:25,146 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:25,146 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/0285a903edec42f8b4082e34c122681b is 50, key is test_row_0/A:col10/1732038564180/Put/seqid=0 2024-11-19T17:49:25,147 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:25,148 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/78abdb577d4342368f35b1dcf758740e is 50, key is test_row_0/B:col10/1732038564180/Put/seqid=0 2024-11-19T17:49:25,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742148_1324 (size=13017) 2024-11-19T17:49:25,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742149_1325 (size=13017) 2024-11-19T17:49:25,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T17:49:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:25,161 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T17:49:25,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/8dbbbfc00e91416f86c8e4ab780f7ae0 is 50, key is test_row_0/A:col10/1732038564204/Put/seqid=0 2024-11-19T17:49:25,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742150_1326 (size=12301) 2024-11-19T17:49:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:25,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:49:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038625406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038625407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:25,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038625408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:25,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038625518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038625518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038625518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:25,559 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/0285a903edec42f8b4082e34c122681b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/0285a903edec42f8b4082e34c122681b 2024-11-19T17:49:25,559 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/78abdb577d4342368f35b1dcf758740e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/78abdb577d4342368f35b1dcf758740e 2024-11-19T17:49:25,569 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/8dbbbfc00e91416f86c8e4ab780f7ae0 2024-11-19T17:49:25,570 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into 0285a903edec42f8b4082e34c122681b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:25,571 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:25,571 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=13, startTime=1732038565130; duration=0sec 2024-11-19T17:49:25,571 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:25,571 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:25,571 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:25,572 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into 78abdb577d4342368f35b1dcf758740e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:25,572 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:25,572 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=13, startTime=1732038565130; duration=0sec 2024-11-19T17:49:25,572 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:25,572 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:25,573 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:25,573 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:25,573 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:25,573 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0c8e54ffcc2144e491c21653d20b8f7a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/f88fa40958e343c7a95b6c7b57531f2c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/85121ff20bbf46a2aa80eddfb6990986] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=36.5 K 2024-11-19T17:49:25,573 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8e54ffcc2144e491c21653d20b8f7a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732038561197 2024-11-19T17:49:25,574 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting f88fa40958e343c7a95b6c7b57531f2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732038562368 2024-11-19T17:49:25,574 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85121ff20bbf46a2aa80eddfb6990986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732038563549 2024-11-19T17:49:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/97a8da37050f4e65a0dbab6a6dcde249 is 50, key is test_row_0/B:col10/1732038564204/Put/seqid=0 2024-11-19T17:49:25,581 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:25,581 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/811f078f0ec34fd087a3f9924291cfa8 is 50, key is test_row_0/C:col10/1732038564180/Put/seqid=0 2024-11-19T17:49:25,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742151_1327 (size=12301) 2024-11-19T17:49:25,594 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/97a8da37050f4e65a0dbab6a6dcde249 2024-11-19T17:49:25,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742152_1328 (size=13017) 2024-11-19T17:49:25,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/044e0b9550134e3e9db2afc1afca0af0 is 50, key is test_row_0/C:col10/1732038564204/Put/seqid=0 2024-11-19T17:49:25,611 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/811f078f0ec34fd087a3f9924291cfa8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/811f078f0ec34fd087a3f9924291cfa8 2024-11-19T17:49:25,616 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into 811f078f0ec34fd087a3f9924291cfa8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:25,616 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:25,616 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=13, startTime=1732038565130; duration=0sec 2024-11-19T17:49:25,616 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:25,617 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:25,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742153_1329 (size=12301) 2024-11-19T17:49:25,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038625728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038625728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:25,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038625729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,022 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/044e0b9550134e3e9db2afc1afca0af0 2024-11-19T17:49:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/8dbbbfc00e91416f86c8e4ab780f7ae0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/8dbbbfc00e91416f86c8e4ab780f7ae0 2024-11-19T17:49:26,031 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/8dbbbfc00e91416f86c8e4ab780f7ae0, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T17:49:26,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/97a8da37050f4e65a0dbab6a6dcde249 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/97a8da37050f4e65a0dbab6a6dcde249 2024-11-19T17:49:26,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038626039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038626039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038626039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,047 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/97a8da37050f4e65a0dbab6a6dcde249, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T17:49:26,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/044e0b9550134e3e9db2afc1afca0af0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/044e0b9550134e3e9db2afc1afca0af0 2024-11-19T17:49:26,051 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/044e0b9550134e3e9db2afc1afca0af0, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T17:49:26,052 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for f7aa8054d507e358fea7889f8e6a6f0a in 891ms, sequenceid=313, compaction requested=false 2024-11-19T17:49:26,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:26,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:26,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T17:49:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T17:49:26,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T17:49:26,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-11-19T17:49:26,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.8220 sec 2024-11-19T17:49:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:49:26,389 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:26,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-19T17:49:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:49:26,390 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:26,391 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:26,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:49:26,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:49:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:26,543 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T17:49:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/20dfca4847fe4c0fbba1e04c4b4d0a47 is 50, key is test_row_0/A:col10/1732038565406/Put/seqid=0 2024-11-19T17:49:26,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742154_1330 (size=12301) 2024-11-19T17:49:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:26,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:26,551 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/20dfca4847fe4c0fbba1e04c4b4d0a47 2024-11-19T17:49:26,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8ed8592f37cb48efb545e6bca88c2e73 is 50, key is test_row_0/B:col10/1732038565406/Put/seqid=0 2024-11-19T17:49:26,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742155_1331 (size=12301) 2024-11-19T17:49:26,561 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8ed8592f37cb48efb545e6bca88c2e73 2024-11-19T17:49:26,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/adc8e33effab4f96a40af2a80ebef072 is 50, key is test_row_0/C:col10/1732038565406/Put/seqid=0 2024-11-19T17:49:26,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742156_1332 (size=12301) 2024-11-19T17:49:26,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038626567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038626568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038626569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038626678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038626678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038626678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:49:26,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038626888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038626888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038626888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:26,971 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/adc8e33effab4f96a40af2a80ebef072 2024-11-19T17:49:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/20dfca4847fe4c0fbba1e04c4b4d0a47 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/20dfca4847fe4c0fbba1e04c4b4d0a47 2024-11-19T17:49:26,979 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/20dfca4847fe4c0fbba1e04c4b4d0a47, entries=150, sequenceid=337, filesize=12.0 K 2024-11-19T17:49:26,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8ed8592f37cb48efb545e6bca88c2e73 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ed8592f37cb48efb545e6bca88c2e73 2024-11-19T17:49:26,985 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ed8592f37cb48efb545e6bca88c2e73, entries=150, sequenceid=337, filesize=12.0 K 2024-11-19T17:49:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/adc8e33effab4f96a40af2a80ebef072 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/adc8e33effab4f96a40af2a80ebef072 2024-11-19T17:49:26,989 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/adc8e33effab4f96a40af2a80ebef072, entries=150, sequenceid=337, filesize=12.0 K 2024-11-19T17:49:26,990 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for f7aa8054d507e358fea7889f8e6a6f0a in 447ms, sequenceid=337, compaction requested=true 2024-11-19T17:49:26,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:26,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:26,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-19T17:49:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-19T17:49:26,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T17:49:26,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 600 msec 2024-11-19T17:49:26,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 604 msec 2024-11-19T17:49:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:49:27,019 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:27,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-19T17:49:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:49:27,021 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:27,021 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:27,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:49:27,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T17:49:27,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:27,174 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T17:49:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/7725544f81434981b7d3dcd33f557581 is 50, key is test_row_0/A:col10/1732038566554/Put/seqid=0 2024-11-19T17:49:27,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742157_1333 (size=12301) 2024-11-19T17:49:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:27,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:27,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038627285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038627286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038627288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:49:27,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038627399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038627399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038627399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,580 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/7725544f81434981b7d3dcd33f557581 2024-11-19T17:49:27,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/2fa2abf2c3e14613afafb52d9f93e2be is 50, key is test_row_0/B:col10/1732038566554/Put/seqid=0 2024-11-19T17:49:27,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742158_1334 (size=12301) 2024-11-19T17:49:27,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038627609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038627609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038627609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:49:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038627918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038627918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038627918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:27,989 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/2fa2abf2c3e14613afafb52d9f93e2be 2024-11-19T17:49:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/c1e0344dabbe4edcb9fa82e287a4e6b4 is 50, key is test_row_0/C:col10/1732038566554/Put/seqid=0 2024-11-19T17:49:28,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742159_1335 (size=12301) 2024-11-19T17:49:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:49:28,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56254 deadline: 1732038628278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,279 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,400 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/c1e0344dabbe4edcb9fa82e287a4e6b4 2024-11-19T17:49:28,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/7725544f81434981b7d3dcd33f557581 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7725544f81434981b7d3dcd33f557581 2024-11-19T17:49:28,406 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7725544f81434981b7d3dcd33f557581, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T17:49:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/2fa2abf2c3e14613afafb52d9f93e2be as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2fa2abf2c3e14613afafb52d9f93e2be 2024-11-19T17:49:28,410 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2fa2abf2c3e14613afafb52d9f93e2be, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T17:49:28,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/c1e0344dabbe4edcb9fa82e287a4e6b4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c1e0344dabbe4edcb9fa82e287a4e6b4 2024-11-19T17:49:28,414 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c1e0344dabbe4edcb9fa82e287a4e6b4, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T17:49:28,415 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f7aa8054d507e358fea7889f8e6a6f0a in 1240ms, sequenceid=352, compaction requested=true 2024-11-19T17:49:28,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:28,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:28,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-19T17:49:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-19T17:49:28,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T17:49:28,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3950 sec 2024-11-19T17:49:28,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.3990 sec 2024-11-19T17:49:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:28,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:28,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T17:49:28,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/397f705d9dda4ce38355bc2476e8cf64 is 50, key is test_row_0/A:col10/1732038567284/Put/seqid=0 2024-11-19T17:49:28,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742160_1336 (size=12301) 2024-11-19T17:49:28,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038628447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038628447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038628449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038628559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038628559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038628559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038628769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038628769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038628769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:28,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/397f705d9dda4ce38355bc2476e8cf64 2024-11-19T17:49:28,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/44246d7373404edbbc30be08d715465f is 50, key is test_row_0/B:col10/1732038567284/Put/seqid=0 2024-11-19T17:49:28,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742161_1337 (size=12301) 2024-11-19T17:49:29,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038629078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:29,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038629078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:29,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038629078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:49:29,168 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:29,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-19T17:49:29,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:49:29,171 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:29,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:29,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/44246d7373404edbbc30be08d715465f 2024-11-19T17:49:29,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/3de0df7ed2704d17b8cf89d14ce261b1 is 50, key is test_row_0/C:col10/1732038567284/Put/seqid=0 2024-11-19T17:49:29,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742162_1338 (size=12301) 2024-11-19T17:49:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:49:29,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:49:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:49:29,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:29,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:49:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038629588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:29,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038629588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:29,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038629588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:29,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:49:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:29,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/3de0df7ed2704d17b8cf89d14ce261b1 2024-11-19T17:49:29,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/397f705d9dda4ce38355bc2476e8cf64 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/397f705d9dda4ce38355bc2476e8cf64 2024-11-19T17:49:29,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/397f705d9dda4ce38355bc2476e8cf64, entries=150, sequenceid=375, filesize=12.0 K 2024-11-19T17:49:29,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/44246d7373404edbbc30be08d715465f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/44246d7373404edbbc30be08d715465f 2024-11-19T17:49:29,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/44246d7373404edbbc30be08d715465f, entries=150, sequenceid=375, filesize=12.0 K 2024-11-19T17:49:29,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/3de0df7ed2704d17b8cf89d14ce261b1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/3de0df7ed2704d17b8cf89d14ce261b1 2024-11-19T17:49:29,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/3de0df7ed2704d17b8cf89d14ce261b1, entries=150, sequenceid=375, filesize=12.0 K 2024-11-19T17:49:29,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f7aa8054d507e358fea7889f8e6a6f0a in 1247ms, sequenceid=375, compaction requested=true 2024-11-19T17:49:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:29,678 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:49:29,678 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:49:29,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:29,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:29,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:29,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:29,680 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:49:29,680 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:49:29,680 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:29,680 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:29,680 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,680 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,680 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/0285a903edec42f8b4082e34c122681b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/8dbbbfc00e91416f86c8e4ab780f7ae0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/20dfca4847fe4c0fbba1e04c4b4d0a47, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7725544f81434981b7d3dcd33f557581, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/397f705d9dda4ce38355bc2476e8cf64] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=60.8 K 2024-11-19T17:49:29,680 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/78abdb577d4342368f35b1dcf758740e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/97a8da37050f4e65a0dbab6a6dcde249, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ed8592f37cb48efb545e6bca88c2e73, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2fa2abf2c3e14613afafb52d9f93e2be, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/44246d7373404edbbc30be08d715465f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=60.8 K 2024-11-19T17:49:29,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:49:29,681 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 78abdb577d4342368f35b1dcf758740e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732038563549 2024-11-19T17:49:29,681 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0285a903edec42f8b4082e34c122681b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732038563549 2024-11-19T17:49:29,681 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 97a8da37050f4e65a0dbab6a6dcde249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732038564198 2024-11-19T17:49:29,681 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dbbbfc00e91416f86c8e4ab780f7ae0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732038564198 2024-11-19T17:49:29,681 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20dfca4847fe4c0fbba1e04c4b4d0a47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732038565397 2024-11-19T17:49:29,681 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed8592f37cb48efb545e6bca88c2e73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732038565397 2024-11-19T17:49:29,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7725544f81434981b7d3dcd33f557581, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732038566554 2024-11-19T17:49:29,682 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa2abf2c3e14613afafb52d9f93e2be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732038566554 2024-11-19T17:49:29,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 397f705d9dda4ce38355bc2476e8cf64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732038567284 2024-11-19T17:49:29,682 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 44246d7373404edbbc30be08d715465f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732038567284 2024-11-19T17:49:29,691 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:29,691 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/1d1fe5b3160945f3aaedb5b95146e817 is 50, key is test_row_0/A:col10/1732038567284/Put/seqid=0 2024-11-19T17:49:29,691 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:29,691 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/0b7b69290dcf4fbfb1de8e1a55b314a6 is 50, key is test_row_0/B:col10/1732038567284/Put/seqid=0 2024-11-19T17:49:29,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742163_1339 (size=13187) 2024-11-19T17:49:29,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742164_1340 (size=13187) 2024-11-19T17:49:29,708 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/0b7b69290dcf4fbfb1de8e1a55b314a6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0b7b69290dcf4fbfb1de8e1a55b314a6 2024-11-19T17:49:29,713 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into 0b7b69290dcf4fbfb1de8e1a55b314a6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:29,713 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:29,713 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=11, startTime=1732038569678; duration=0sec 2024-11-19T17:49:29,713 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:29,713 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:29,713 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:49:29,714 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:49:29,714 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:29,714 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,714 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/811f078f0ec34fd087a3f9924291cfa8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/044e0b9550134e3e9db2afc1afca0af0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/adc8e33effab4f96a40af2a80ebef072, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c1e0344dabbe4edcb9fa82e287a4e6b4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/3de0df7ed2704d17b8cf89d14ce261b1] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=60.8 K 2024-11-19T17:49:29,715 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 811f078f0ec34fd087a3f9924291cfa8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732038563549 2024-11-19T17:49:29,715 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 044e0b9550134e3e9db2afc1afca0af0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732038564198 2024-11-19T17:49:29,715 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting adc8e33effab4f96a40af2a80ebef072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732038565397 2024-11-19T17:49:29,715 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c1e0344dabbe4edcb9fa82e287a4e6b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732038566554 2024-11-19T17:49:29,716 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de0df7ed2704d17b8cf89d14ce261b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732038567284 2024-11-19T17:49:29,728 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#293 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:29,729 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/1595c0a2e38b41648172266d44e36e74 is 50, key is test_row_0/C:col10/1732038567284/Put/seqid=0 2024-11-19T17:49:29,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742165_1341 (size=13187) 2024-11-19T17:49:29,749 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/1595c0a2e38b41648172266d44e36e74 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/1595c0a2e38b41648172266d44e36e74 2024-11-19T17:49:29,754 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into 1595c0a2e38b41648172266d44e36e74(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:29,754 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:29,754 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=11, startTime=1732038569679; duration=0sec 2024-11-19T17:49:29,754 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:29,754 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:29,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:49:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:29,781 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:49:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/7619f4d1f59e4689876c37327c3f06eb is 50, key is test_row_0/A:col10/1732038568446/Put/seqid=0 2024-11-19T17:49:29,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742166_1342 (size=12301) 2024-11-19T17:49:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:49:30,106 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/1d1fe5b3160945f3aaedb5b95146e817 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1d1fe5b3160945f3aaedb5b95146e817 2024-11-19T17:49:30,110 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into 1d1fe5b3160945f3aaedb5b95146e817(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:30,110 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:30,110 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=11, startTime=1732038569678; duration=0sec 2024-11-19T17:49:30,111 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:30,111 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:30,189 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/7619f4d1f59e4689876c37327c3f06eb 2024-11-19T17:49:30,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8ecc3007ca9c486a9feb4fa9b7fe1855 is 50, key is test_row_0/B:col10/1732038568446/Put/seqid=0 2024-11-19T17:49:30,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742167_1343 (size=12301) 2024-11-19T17:49:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:49:30,600 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8ecc3007ca9c486a9feb4fa9b7fe1855 2024-11-19T17:49:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:30,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/bc3447371b4c473984119d456a6b5d0d is 50, key is test_row_0/C:col10/1732038568446/Put/seqid=0 2024-11-19T17:49:30,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742168_1344 (size=12301) 2024-11-19T17:49:30,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038630626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038630628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:30,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038630630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:30,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038630739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038630739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:30,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038630740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038630948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038630949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038630949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,010 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/bc3447371b4c473984119d456a6b5d0d 2024-11-19T17:49:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/7619f4d1f59e4689876c37327c3f06eb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7619f4d1f59e4689876c37327c3f06eb 2024-11-19T17:49:31,017 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7619f4d1f59e4689876c37327c3f06eb, entries=150, sequenceid=390, filesize=12.0 K 2024-11-19T17:49:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/8ecc3007ca9c486a9feb4fa9b7fe1855 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ecc3007ca9c486a9feb4fa9b7fe1855 2024-11-19T17:49:31,022 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ecc3007ca9c486a9feb4fa9b7fe1855, entries=150, sequenceid=390, filesize=12.0 K 2024-11-19T17:49:31,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/bc3447371b4c473984119d456a6b5d0d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/bc3447371b4c473984119d456a6b5d0d 2024-11-19T17:49:31,028 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/bc3447371b4c473984119d456a6b5d0d, entries=150, sequenceid=390, filesize=12.0 K 2024-11-19T17:49:31,029 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7aa8054d507e358fea7889f8e6a6f0a in 1248ms, sequenceid=390, compaction requested=false 2024-11-19T17:49:31,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:31,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-19T17:49:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-19T17:49:31,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T17:49:31,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-19T17:49:31,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.8660 sec 2024-11-19T17:49:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:31,261 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:31,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:49:31,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/864ff51d8bce43f8b4dce7fad88fd89a is 50, key is test_row_0/A:col10/1732038570627/Put/seqid=0 2024-11-19T17:49:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742169_1345 (size=17181) 2024-11-19T17:49:31,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038631271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038631272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038631274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:49:31,319 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:31,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-19T17:49:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:49:31,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:31,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:31,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:31,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038631378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038631378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038631378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:49:31,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038631588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038631588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038631588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:49:31,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/864ff51d8bce43f8b4dce7fad88fd89a 2024-11-19T17:49:31,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/3af3cfb9e6904092b7da322c5bfbf694 is 50, key is test_row_0/B:col10/1732038570627/Put/seqid=0 2024-11-19T17:49:31,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742170_1346 (size=12301) 2024-11-19T17:49:31,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:31,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:31,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038631899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038631899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038631899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:31,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:31,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:31,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:31,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:49:32,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/3af3cfb9e6904092b7da322c5bfbf694 2024-11-19T17:49:32,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/0a6904df278b478788e7eab307e11b2a is 50, key is test_row_0/C:col10/1732038570627/Put/seqid=0 2024-11-19T17:49:32,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742171_1347 (size=12301) 2024-11-19T17:49:32,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/0a6904df278b478788e7eab307e11b2a 2024-11-19T17:49:32,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/864ff51d8bce43f8b4dce7fad88fd89a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/864ff51d8bce43f8b4dce7fad88fd89a 2024-11-19T17:49:32,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/864ff51d8bce43f8b4dce7fad88fd89a, entries=250, sequenceid=417, filesize=16.8 K 2024-11-19T17:49:32,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/3af3cfb9e6904092b7da322c5bfbf694 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/3af3cfb9e6904092b7da322c5bfbf694 2024-11-19T17:49:32,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/3af3cfb9e6904092b7da322c5bfbf694, entries=150, sequenceid=417, filesize=12.0 K 2024-11-19T17:49:32,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/0a6904df278b478788e7eab307e11b2a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0a6904df278b478788e7eab307e11b2a 2024-11-19T17:49:32,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0a6904df278b478788e7eab307e11b2a, entries=150, sequenceid=417, filesize=12.0 K 2024-11-19T17:49:32,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f7aa8054d507e358fea7889f8e6a6f0a in 852ms, sequenceid=417, compaction requested=true 2024-11-19T17:49:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:32,113 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:32,113 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7aa8054d507e358fea7889f8e6a6f0a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:32,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:32,114 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:32,114 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:32,114 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/B is initiating minor compaction (all files) 2024-11-19T17:49:32,114 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/A is initiating minor compaction (all files) 2024-11-19T17:49:32,114 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/B in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,114 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/A in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,114 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0b7b69290dcf4fbfb1de8e1a55b314a6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ecc3007ca9c486a9feb4fa9b7fe1855, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/3af3cfb9e6904092b7da322c5bfbf694] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=36.9 K 2024-11-19T17:49:32,114 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1d1fe5b3160945f3aaedb5b95146e817, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7619f4d1f59e4689876c37327c3f06eb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/864ff51d8bce43f8b4dce7fad88fd89a] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=41.7 K 2024-11-19T17:49:32,114 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7b69290dcf4fbfb1de8e1a55b314a6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732038567284 2024-11-19T17:49:32,114 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d1fe5b3160945f3aaedb5b95146e817, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732038567284 2024-11-19T17:49:32,115 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ecc3007ca9c486a9feb4fa9b7fe1855, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732038568437 2024-11-19T17:49:32,115 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7619f4d1f59e4689876c37327c3f06eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732038568437 2024-11-19T17:49:32,115 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af3cfb9e6904092b7da322c5bfbf694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732038570627 2024-11-19T17:49:32,115 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 864ff51d8bce43f8b4dce7fad88fd89a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732038570627 2024-11-19T17:49:32,121 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:32,122 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#A#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:32,123 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/6ae0b2c1532843faa259d5eee8c767c2 is 50, key is test_row_0/B:col10/1732038570627/Put/seqid=0 2024-11-19T17:49:32,123 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/db1c565ad1db435483576f622cb02ad0 is 50, key is test_row_0/A:col10/1732038570627/Put/seqid=0 2024-11-19T17:49:32,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742172_1348 (size=13289) 2024-11-19T17:49:32,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742173_1349 (size=13289) 2024-11-19T17:49:32,144 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/db1c565ad1db435483576f622cb02ad0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/db1c565ad1db435483576f622cb02ad0 2024-11-19T17:49:32,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/A of f7aa8054d507e358fea7889f8e6a6f0a into db1c565ad1db435483576f622cb02ad0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:32,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:32,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/A, priority=13, startTime=1732038572113; duration=0sec 2024-11-19T17:49:32,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:32,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:A 2024-11-19T17:49:32,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:32,148 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:32,148 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): f7aa8054d507e358fea7889f8e6a6f0a/C is initiating minor compaction (all files) 2024-11-19T17:49:32,148 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7aa8054d507e358fea7889f8e6a6f0a/C in TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,148 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/1595c0a2e38b41648172266d44e36e74, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/bc3447371b4c473984119d456a6b5d0d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0a6904df278b478788e7eab307e11b2a] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp, totalSize=36.9 K 2024-11-19T17:49:32,149 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1595c0a2e38b41648172266d44e36e74, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732038567284 2024-11-19T17:49:32,149 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc3447371b4c473984119d456a6b5d0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732038568437 2024-11-19T17:49:32,149 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a6904df278b478788e7eab307e11b2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732038570627 2024-11-19T17:49:32,156 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7aa8054d507e358fea7889f8e6a6f0a#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:32,156 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/83f2acd60b8947dcade6934359a40df7 is 50, key is test_row_0/C:col10/1732038570627/Put/seqid=0 2024-11-19T17:49:32,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742174_1350 (size=13289) 2024-11-19T17:49:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:32,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:32,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:49:32,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/d2cf576c779a42808b86cbb3edf4d93b is 50, key is test_row_0/A:col10/1732038571270/Put/seqid=0 2024-11-19T17:49:32,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742175_1351 (size=12297) 2024-11-19T17:49:32,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:32,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038632277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:32,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:32,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:32,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038632389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038632408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:32,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038632408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038632408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:49:32,526 INFO [Thread-1214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1214. 2024-11-19T17:49:32,526 DEBUG [Thread-1214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:32,526 DEBUG [Thread-1214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,526 INFO [Thread-1220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1220. 2024-11-19T17:49:32,527 INFO [Thread-1218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1218. 2024-11-19T17:49:32,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:32,527 DEBUG [Thread-1218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:32,527 DEBUG [Thread-1220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:32,527 DEBUG [Thread-1218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,527 DEBUG [Thread-1220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,527 DEBUG [Thread-1220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:32,527 DEBUG [Thread-1214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,527 DEBUG [Thread-1218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:32,529 INFO [Thread-1216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1216. 2024-11-19T17:49:32,529 DEBUG [Thread-1216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:32,529 DEBUG [Thread-1216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,529 DEBUG [Thread-1216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:32,530 INFO [Thread-1212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1212. 2024-11-19T17:49:32,530 DEBUG [Thread-1212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:32,530 DEBUG [Thread-1212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,530 DEBUG [Thread-1212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:32,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:32,533 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/6ae0b2c1532843faa259d5eee8c767c2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6ae0b2c1532843faa259d5eee8c767c2 2024-11-19T17:49:32,537 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/B of f7aa8054d507e358fea7889f8e6a6f0a into 6ae0b2c1532843faa259d5eee8c767c2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:32,537 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:32,537 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/B, priority=13, startTime=1732038572113; duration=0sec 2024-11-19T17:49:32,537 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:32,537 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:B 2024-11-19T17:49:32,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:32,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:32,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,566 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/83f2acd60b8947dcade6934359a40df7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/83f2acd60b8947dcade6934359a40df7 2024-11-19T17:49:32,572 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7aa8054d507e358fea7889f8e6a6f0a/C of f7aa8054d507e358fea7889f8e6a6f0a into 83f2acd60b8947dcade6934359a40df7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:32,572 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:32,572 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., storeName=f7aa8054d507e358fea7889f8e6a6f0a/C, priority=13, startTime=1732038572113; duration=0sec 2024-11-19T17:49:32,572 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:32,573 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7aa8054d507e358fea7889f8e6a6f0a:C 2024-11-19T17:49:32,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/d2cf576c779a42808b86cbb3edf4d93b 2024-11-19T17:49:32,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/2a2f833fe1294d61b2c37052bedb0c8c is 50, key is test_row_0/B:col10/1732038571270/Put/seqid=0 2024-11-19T17:49:32,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038632599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:32,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742176_1352 (size=9857) 2024-11-19T17:49:32,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:32,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:32,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:32,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:32,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038632909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:33,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/2a2f833fe1294d61b2c37052bedb0c8c 2024-11-19T17:49:33,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:33,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:33,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:33,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:33,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/b68a0d2cc0834b1fa4730bea6d05dc9a is 50, key is test_row_0/C:col10/1732038571270/Put/seqid=0 2024-11-19T17:49:33,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742177_1353 (size=9857) 2024-11-19T17:49:33,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:33,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:33,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:33,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:33,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:33,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:33,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:33,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:33,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:33,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56262 deadline: 1732038633419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:33,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:33,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56216 deadline: 1732038633419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1732038633419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:33,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56252 deadline: 1732038633419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:33,421 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2146 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:33,421 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:33,421 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in f7aa8054d507e358fea7889f8e6a6f0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:49:33,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/b68a0d2cc0834b1fa4730bea6d05dc9a 2024-11-19T17:49:33,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/d2cf576c779a42808b86cbb3edf4d93b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2cf576c779a42808b86cbb3edf4d93b 2024-11-19T17:49:33,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2cf576c779a42808b86cbb3edf4d93b, entries=150, sequenceid=429, filesize=12.0 K 2024-11-19T17:49:33,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/2a2f833fe1294d61b2c37052bedb0c8c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2a2f833fe1294d61b2c37052bedb0c8c 2024-11-19T17:49:33,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2a2f833fe1294d61b2c37052bedb0c8c, entries=100, sequenceid=429, filesize=9.6 K 2024-11-19T17:49:33,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/b68a0d2cc0834b1fa4730bea6d05dc9a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/b68a0d2cc0834b1fa4730bea6d05dc9a 2024-11-19T17:49:33,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/b68a0d2cc0834b1fa4730bea6d05dc9a, entries=100, sequenceid=429, filesize=9.6 K 2024-11-19T17:49:33,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f7aa8054d507e358fea7889f8e6a6f0a in 1260ms, sequenceid=429, compaction requested=false 2024-11-19T17:49:33,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:49:33,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:49:33,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:33,482 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:49:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/5f008d82c5bf41109c3c97e17ef7a281 is 50, key is test_row_0/A:col10/1732038572268/Put/seqid=0 2024-11-19T17:49:33,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742178_1354 (size=12301) 2024-11-19T17:49:33,894 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/5f008d82c5bf41109c3c97e17ef7a281 2024-11-19T17:49:33,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f376dc002356443b9349ac28ff5aacde is 50, key is test_row_0/B:col10/1732038572268/Put/seqid=0 2024-11-19T17:49:33,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742179_1355 (size=12301) 2024-11-19T17:49:34,312 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f376dc002356443b9349ac28ff5aacde 2024-11-19T17:49:34,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/75840f34454f4b3ba7280d996475ac17 is 50, key is test_row_0/C:col10/1732038572268/Put/seqid=0 2024-11-19T17:49:34,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742180_1356 (size=12301) 2024-11-19T17:49:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:34,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. as already flushing 2024-11-19T17:49:34,442 INFO [Thread-1205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1205. 2024-11-19T17:49:34,442 DEBUG [Thread-1205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:34,442 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:34,442 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:34,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:34,734 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/75840f34454f4b3ba7280d996475ac17 2024-11-19T17:49:34,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/5f008d82c5bf41109c3c97e17ef7a281 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f008d82c5bf41109c3c97e17ef7a281 2024-11-19T17:49:34,751 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f008d82c5bf41109c3c97e17ef7a281, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T17:49:34,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/f376dc002356443b9349ac28ff5aacde as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f376dc002356443b9349ac28ff5aacde 2024-11-19T17:49:34,757 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f376dc002356443b9349ac28ff5aacde, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T17:49:34,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/75840f34454f4b3ba7280d996475ac17 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/75840f34454f4b3ba7280d996475ac17 2024-11-19T17:49:34,761 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/75840f34454f4b3ba7280d996475ac17, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T17:49:34,762 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=6.71 KB/6870 for f7aa8054d507e358fea7889f8e6a6f0a in 1280ms, sequenceid=456, compaction requested=true 2024-11-19T17:49:34,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for f7aa8054d507e358fea7889f8e6a6f0a: 2024-11-19T17:49:34,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:34,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-19T17:49:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-19T17:49:34,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T17:49:34,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4420 sec 2024-11-19T17:49:34,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 3.4460 sec 2024-11-19T17:49:35,431 INFO [Thread-1209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1209. 2024-11-19T17:49:35,431 DEBUG [Thread-1209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:35,431 DEBUG [Thread-1209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:35,431 DEBUG [Thread-1209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:35,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:35,441 INFO [Thread-1203 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1203. 2024-11-19T17:49:35,441 DEBUG [Thread-1203 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:35,441 DEBUG [Thread-1203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:35,441 DEBUG [Thread-1203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:35,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:35,452 INFO [Thread-1201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1201. 2024-11-19T17:49:35,452 DEBUG [Thread-1201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:35,452 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:35,452 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:35,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:49:35,480 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:38,300 INFO [Thread-1207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1207. 2024-11-19T17:49:38,300 DEBUG [Thread-1207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:49:38,300 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:38,300 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:38,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:38,300 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:49:38,300 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-19T17:49:38,300 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-19T17:49:38,300 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3565 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3518 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3528 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3569 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3524 2024-11-19T17:49:38,301 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:49:38,301 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:49:38,301 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:49:38,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:38,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:38,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:49:38,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:49:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:49:38,305 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038578305"}]},"ts":"1732038578305"} 2024-11-19T17:49:38,307 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:49:38,307 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:49:38,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:49:38,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, UNASSIGN}] 2024-11-19T17:49:38,310 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, UNASSIGN 2024-11-19T17:49:38,311 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=f7aa8054d507e358fea7889f8e6a6f0a, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:38,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, UNASSIGN because future has completed 2024-11-19T17:49:38,312 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:49:38,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:49:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:49:38,465 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:38,465 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:49:38,465 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing f7aa8054d507e358fea7889f8e6a6f0a, disabling compactions & flushes 2024-11-19T17:49:38,466 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:38,466 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:38,466 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. after waiting 0 ms 2024-11-19T17:49:38,466 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:38,466 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing f7aa8054d507e358fea7889f8e6a6f0a 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T17:49:38,472 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/41b1404c2a2e437fb609a4a203ca8944 is 50, key is test_row_0/A:col10/1732038578299/Put/seqid=0 2024-11-19T17:49:38,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742181_1357 (size=12301) 2024-11-19T17:49:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:49:38,877 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/41b1404c2a2e437fb609a4a203ca8944 2024-11-19T17:49:38,890 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/655f8b04693048c8a6d44b61c5302901 is 50, key is test_row_0/B:col10/1732038578299/Put/seqid=0 2024-11-19T17:49:38,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742182_1358 (size=12301) 2024-11-19T17:49:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:49:39,294 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/655f8b04693048c8a6d44b61c5302901 2024-11-19T17:49:39,301 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/8e1b421644d04f8fbbfbe6a3a2efdbe6 is 50, key is test_row_0/C:col10/1732038578299/Put/seqid=0 2024-11-19T17:49:39,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742183_1359 (size=12301) 2024-11-19T17:49:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:49:39,705 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/8e1b421644d04f8fbbfbe6a3a2efdbe6 2024-11-19T17:49:39,709 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/A/41b1404c2a2e437fb609a4a203ca8944 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/41b1404c2a2e437fb609a4a203ca8944 2024-11-19T17:49:39,712 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/41b1404c2a2e437fb609a4a203ca8944, entries=150, sequenceid=464, filesize=12.0 K 2024-11-19T17:49:39,713 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/B/655f8b04693048c8a6d44b61c5302901 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/655f8b04693048c8a6d44b61c5302901 2024-11-19T17:49:39,716 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/655f8b04693048c8a6d44b61c5302901, entries=150, sequenceid=464, filesize=12.0 K 2024-11-19T17:49:39,717 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/.tmp/C/8e1b421644d04f8fbbfbe6a3a2efdbe6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8e1b421644d04f8fbbfbe6a3a2efdbe6 2024-11-19T17:49:39,719 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8e1b421644d04f8fbbfbe6a3a2efdbe6, entries=150, sequenceid=464, filesize=12.0 K 2024-11-19T17:49:39,720 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f7aa8054d507e358fea7889f8e6a6f0a in 1254ms, sequenceid=464, compaction requested=true 2024-11-19T17:49:39,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/49b52b0be6a0439395597a9e94fc0341, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2a97f695fbc44d9805d82b74327db3c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/610a7b2a46734f16998b65c0ecd89d5c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/450a65029cc540e5a953a3f94eda69f7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/669ab5429f934646adfb4a728a1344dd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3e25b34fff4f44e68b4530c39f1747d0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7d52017d6c5b43e18b7f12e428570fb6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c878b2d098f94cc59fcff771db1113b8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f02d679673a4fe29f4e72ea6456106d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ef09ff920ee847389d6df8e66c71a830, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ae1d928d274d42898ce77efc6d36e6ae, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d650433b26a04572867c7517c35cc980, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1f62b2fc7839441fae99f41bf0ca30c3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/97a78895520944a5812fc6fa550f5eec, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b67450a4a9bb46a387bdacd5a9a4d60f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/2a6b066f82da476db54db87bdfb0e969, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c6a648480ee9486892ef5491f275ee77, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3cdf59437c69422eab8bb87200c0af8f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/a9935fd124be4edc99cb2a37bcb766c9, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b1e56598f5754f3abf2465d8b521b571, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/95260de172024e948db545b91b0235ae, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/0285a903edec42f8b4082e34c122681b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/8dbbbfc00e91416f86c8e4ab780f7ae0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/20dfca4847fe4c0fbba1e04c4b4d0a47, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7725544f81434981b7d3dcd33f557581, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1d1fe5b3160945f3aaedb5b95146e817, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/397f705d9dda4ce38355bc2476e8cf64, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7619f4d1f59e4689876c37327c3f06eb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/864ff51d8bce43f8b4dce7fad88fd89a] to archive 2024-11-19T17:49:39,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:49:39,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/49b52b0be6a0439395597a9e94fc0341 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/49b52b0be6a0439395597a9e94fc0341 2024-11-19T17:49:39,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2a97f695fbc44d9805d82b74327db3c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2a97f695fbc44d9805d82b74327db3c 2024-11-19T17:49:39,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/610a7b2a46734f16998b65c0ecd89d5c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/610a7b2a46734f16998b65c0ecd89d5c 2024-11-19T17:49:39,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/450a65029cc540e5a953a3f94eda69f7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/450a65029cc540e5a953a3f94eda69f7 2024-11-19T17:49:39,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/669ab5429f934646adfb4a728a1344dd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/669ab5429f934646adfb4a728a1344dd 2024-11-19T17:49:39,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3e25b34fff4f44e68b4530c39f1747d0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3e25b34fff4f44e68b4530c39f1747d0 2024-11-19T17:49:39,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7d52017d6c5b43e18b7f12e428570fb6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7d52017d6c5b43e18b7f12e428570fb6 2024-11-19T17:49:39,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c878b2d098f94cc59fcff771db1113b8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c878b2d098f94cc59fcff771db1113b8 2024-11-19T17:49:39,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f02d679673a4fe29f4e72ea6456106d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f02d679673a4fe29f4e72ea6456106d 2024-11-19T17:49:39,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ef09ff920ee847389d6df8e66c71a830 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ef09ff920ee847389d6df8e66c71a830 2024-11-19T17:49:39,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ae1d928d274d42898ce77efc6d36e6ae to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/ae1d928d274d42898ce77efc6d36e6ae 2024-11-19T17:49:39,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d650433b26a04572867c7517c35cc980 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d650433b26a04572867c7517c35cc980 2024-11-19T17:49:39,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1f62b2fc7839441fae99f41bf0ca30c3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1f62b2fc7839441fae99f41bf0ca30c3 2024-11-19T17:49:39,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/97a78895520944a5812fc6fa550f5eec to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/97a78895520944a5812fc6fa550f5eec 2024-11-19T17:49:39,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b67450a4a9bb46a387bdacd5a9a4d60f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b67450a4a9bb46a387bdacd5a9a4d60f 2024-11-19T17:49:39,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/2a6b066f82da476db54db87bdfb0e969 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/2a6b066f82da476db54db87bdfb0e969 2024-11-19T17:49:39,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c6a648480ee9486892ef5491f275ee77 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/c6a648480ee9486892ef5491f275ee77 2024-11-19T17:49:39,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3cdf59437c69422eab8bb87200c0af8f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/3cdf59437c69422eab8bb87200c0af8f 2024-11-19T17:49:39,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/a9935fd124be4edc99cb2a37bcb766c9 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/a9935fd124be4edc99cb2a37bcb766c9 2024-11-19T17:49:39,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b1e56598f5754f3abf2465d8b521b571 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/b1e56598f5754f3abf2465d8b521b571 2024-11-19T17:49:39,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/95260de172024e948db545b91b0235ae to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/95260de172024e948db545b91b0235ae 2024-11-19T17:49:39,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/0285a903edec42f8b4082e34c122681b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/0285a903edec42f8b4082e34c122681b 2024-11-19T17:49:39,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/8dbbbfc00e91416f86c8e4ab780f7ae0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/8dbbbfc00e91416f86c8e4ab780f7ae0 2024-11-19T17:49:39,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/20dfca4847fe4c0fbba1e04c4b4d0a47 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/20dfca4847fe4c0fbba1e04c4b4d0a47 2024-11-19T17:49:39,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7725544f81434981b7d3dcd33f557581 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7725544f81434981b7d3dcd33f557581 2024-11-19T17:49:39,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1d1fe5b3160945f3aaedb5b95146e817 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/1d1fe5b3160945f3aaedb5b95146e817 2024-11-19T17:49:39,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/397f705d9dda4ce38355bc2476e8cf64 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/397f705d9dda4ce38355bc2476e8cf64 2024-11-19T17:49:39,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7619f4d1f59e4689876c37327c3f06eb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/7619f4d1f59e4689876c37327c3f06eb 2024-11-19T17:49:39,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/864ff51d8bce43f8b4dce7fad88fd89a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/864ff51d8bce43f8b4dce7fad88fd89a 2024-11-19T17:49:39,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/59a66c219831455bbecd474cbde0817d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8b73216d25c54d5b84bf7bb9586e9e17, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e4b4f16382f0499d8e0df52b75e12077, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e5b9344cffa74745803b6954d7c667e8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/843f7b156694429eb65d76130c9e5ca5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0af9f6268db04272a83b286a09573644, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f53bd119b1084e4bbcd2b6e39c6785ed, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/c11e0b3f20ab4d28a0359f6c7cdcc34d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/a6c627a5a80c46bdb276012d3c686b7a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/55e6f67418c04704ad1c9cfe077b0bc5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6e171de3268a430eb43b68acb828e0cd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f7ad8b1553bd4fee8a5925bc8a12fc9d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f5bfb7bbc97445a39a9b8dc06e21e60e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/53807495ba4d4412b41aed846cd55f88, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/26b06eda72394745a3274dd0ea865440, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/47546fa1da7941dfb12485a33a4dbc1e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/34dba89f6eb54f9c96e3de1027c6de24, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/999b4ed0d13d420985d13dc57aaa266e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/cda466ad4bd34469a41eef3cce379676, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/7b3b06783f7349bc9b45f288ecd2c3d6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/78abdb577d4342368f35b1dcf758740e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/ba981e1a7b494cecbff5fdcf569e379d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/97a8da37050f4e65a0dbab6a6dcde249, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ed8592f37cb48efb545e6bca88c2e73, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2fa2abf2c3e14613afafb52d9f93e2be, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0b7b69290dcf4fbfb1de8e1a55b314a6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/44246d7373404edbbc30be08d715465f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ecc3007ca9c486a9feb4fa9b7fe1855, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/3af3cfb9e6904092b7da322c5bfbf694] to archive 2024-11-19T17:49:39,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:49:39,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/59a66c219831455bbecd474cbde0817d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/59a66c219831455bbecd474cbde0817d 2024-11-19T17:49:39,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8b73216d25c54d5b84bf7bb9586e9e17 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8b73216d25c54d5b84bf7bb9586e9e17 2024-11-19T17:49:39,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e4b4f16382f0499d8e0df52b75e12077 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e4b4f16382f0499d8e0df52b75e12077 2024-11-19T17:49:39,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e5b9344cffa74745803b6954d7c667e8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/e5b9344cffa74745803b6954d7c667e8 2024-11-19T17:49:39,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/843f7b156694429eb65d76130c9e5ca5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/843f7b156694429eb65d76130c9e5ca5 2024-11-19T17:49:39,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0af9f6268db04272a83b286a09573644 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0af9f6268db04272a83b286a09573644 2024-11-19T17:49:39,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f53bd119b1084e4bbcd2b6e39c6785ed to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f53bd119b1084e4bbcd2b6e39c6785ed 2024-11-19T17:49:39,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/c11e0b3f20ab4d28a0359f6c7cdcc34d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/c11e0b3f20ab4d28a0359f6c7cdcc34d 2024-11-19T17:49:39,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/a6c627a5a80c46bdb276012d3c686b7a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/a6c627a5a80c46bdb276012d3c686b7a 2024-11-19T17:49:39,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/55e6f67418c04704ad1c9cfe077b0bc5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/55e6f67418c04704ad1c9cfe077b0bc5 2024-11-19T17:49:39,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6e171de3268a430eb43b68acb828e0cd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6e171de3268a430eb43b68acb828e0cd 2024-11-19T17:49:39,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f7ad8b1553bd4fee8a5925bc8a12fc9d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f7ad8b1553bd4fee8a5925bc8a12fc9d 2024-11-19T17:49:39,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f5bfb7bbc97445a39a9b8dc06e21e60e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f5bfb7bbc97445a39a9b8dc06e21e60e 2024-11-19T17:49:39,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/53807495ba4d4412b41aed846cd55f88 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/53807495ba4d4412b41aed846cd55f88 2024-11-19T17:49:39,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/26b06eda72394745a3274dd0ea865440 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/26b06eda72394745a3274dd0ea865440 2024-11-19T17:49:39,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/47546fa1da7941dfb12485a33a4dbc1e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/47546fa1da7941dfb12485a33a4dbc1e 2024-11-19T17:49:39,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/34dba89f6eb54f9c96e3de1027c6de24 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/34dba89f6eb54f9c96e3de1027c6de24 2024-11-19T17:49:39,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/999b4ed0d13d420985d13dc57aaa266e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/999b4ed0d13d420985d13dc57aaa266e 2024-11-19T17:49:39,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/cda466ad4bd34469a41eef3cce379676 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/cda466ad4bd34469a41eef3cce379676 2024-11-19T17:49:39,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/7b3b06783f7349bc9b45f288ecd2c3d6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/7b3b06783f7349bc9b45f288ecd2c3d6 2024-11-19T17:49:39,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/78abdb577d4342368f35b1dcf758740e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/78abdb577d4342368f35b1dcf758740e 2024-11-19T17:49:39,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/ba981e1a7b494cecbff5fdcf569e379d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/ba981e1a7b494cecbff5fdcf569e379d 2024-11-19T17:49:39,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/97a8da37050f4e65a0dbab6a6dcde249 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/97a8da37050f4e65a0dbab6a6dcde249 2024-11-19T17:49:39,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ed8592f37cb48efb545e6bca88c2e73 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ed8592f37cb48efb545e6bca88c2e73 2024-11-19T17:49:39,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2fa2abf2c3e14613afafb52d9f93e2be to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2fa2abf2c3e14613afafb52d9f93e2be 2024-11-19T17:49:39,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0b7b69290dcf4fbfb1de8e1a55b314a6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/0b7b69290dcf4fbfb1de8e1a55b314a6 2024-11-19T17:49:39,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/44246d7373404edbbc30be08d715465f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/44246d7373404edbbc30be08d715465f 2024-11-19T17:49:39,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ecc3007ca9c486a9feb4fa9b7fe1855 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/8ecc3007ca9c486a9feb4fa9b7fe1855 2024-11-19T17:49:39,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/3af3cfb9e6904092b7da322c5bfbf694 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/3af3cfb9e6904092b7da322c5bfbf694 2024-11-19T17:49:39,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e09052e0239c46878990b63dc461cad7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39afe135b6924d74b4d7390ca8af1cd2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/90f2f5527c4c40de9345ccf8c2d0b700, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5de7f14d056944c8914b2d91f1e1ebe8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/17df65869cf24c87aa47b6ac03b9653a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e5b5440c40fc42fb8a01a3de19c8821d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5cce96f72fc54449b22917d5e440186f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/9ddcdaba7c554e7eb4b3bcb3fdb50956, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c63fa0417dea41bc8b372b670ef1d40e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ecdee2e805a848c5a1046612dc9a8815, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8be100ca36a3408b8259130d85fae6e4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/388d5e0107cc4641aa7790509c11b1d1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39a674b1950147b38bbf4ddd67a8c267, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ef61b1f74853460980d8c13f9a680ccb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/4976eeb263684cfaa860133be160ef55, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/32035be042864b5c9250f37e9e4a5f4d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/6fddb325d23949bda2ec503fbc307fc2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0c8e54ffcc2144e491c21653d20b8f7a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/daa9af6937aa432ba431727e1bebdc8f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/f88fa40958e343c7a95b6c7b57531f2c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/811f078f0ec34fd087a3f9924291cfa8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/85121ff20bbf46a2aa80eddfb6990986, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/044e0b9550134e3e9db2afc1afca0af0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/adc8e33effab4f96a40af2a80ebef072, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c1e0344dabbe4edcb9fa82e287a4e6b4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/1595c0a2e38b41648172266d44e36e74, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/3de0df7ed2704d17b8cf89d14ce261b1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/bc3447371b4c473984119d456a6b5d0d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0a6904df278b478788e7eab307e11b2a] to archive 2024-11-19T17:49:39,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:49:39,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e09052e0239c46878990b63dc461cad7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e09052e0239c46878990b63dc461cad7 2024-11-19T17:49:39,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39afe135b6924d74b4d7390ca8af1cd2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39afe135b6924d74b4d7390ca8af1cd2 2024-11-19T17:49:39,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/90f2f5527c4c40de9345ccf8c2d0b700 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/90f2f5527c4c40de9345ccf8c2d0b700 2024-11-19T17:49:39,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5de7f14d056944c8914b2d91f1e1ebe8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5de7f14d056944c8914b2d91f1e1ebe8 2024-11-19T17:49:39,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/17df65869cf24c87aa47b6ac03b9653a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/17df65869cf24c87aa47b6ac03b9653a 2024-11-19T17:49:39,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e5b5440c40fc42fb8a01a3de19c8821d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/e5b5440c40fc42fb8a01a3de19c8821d 2024-11-19T17:49:39,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5cce96f72fc54449b22917d5e440186f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/5cce96f72fc54449b22917d5e440186f 2024-11-19T17:49:39,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/9ddcdaba7c554e7eb4b3bcb3fdb50956 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/9ddcdaba7c554e7eb4b3bcb3fdb50956 2024-11-19T17:49:39,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c63fa0417dea41bc8b372b670ef1d40e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c63fa0417dea41bc8b372b670ef1d40e 2024-11-19T17:49:39,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ecdee2e805a848c5a1046612dc9a8815 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ecdee2e805a848c5a1046612dc9a8815 2024-11-19T17:49:39,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8be100ca36a3408b8259130d85fae6e4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8be100ca36a3408b8259130d85fae6e4 2024-11-19T17:49:39,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/388d5e0107cc4641aa7790509c11b1d1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/388d5e0107cc4641aa7790509c11b1d1 2024-11-19T17:49:39,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39a674b1950147b38bbf4ddd67a8c267 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/39a674b1950147b38bbf4ddd67a8c267 2024-11-19T17:49:39,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ef61b1f74853460980d8c13f9a680ccb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/ef61b1f74853460980d8c13f9a680ccb 2024-11-19T17:49:39,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/4976eeb263684cfaa860133be160ef55 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/4976eeb263684cfaa860133be160ef55 2024-11-19T17:49:39,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/32035be042864b5c9250f37e9e4a5f4d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/32035be042864b5c9250f37e9e4a5f4d 2024-11-19T17:49:39,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/6fddb325d23949bda2ec503fbc307fc2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/6fddb325d23949bda2ec503fbc307fc2 2024-11-19T17:49:39,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0c8e54ffcc2144e491c21653d20b8f7a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0c8e54ffcc2144e491c21653d20b8f7a 2024-11-19T17:49:39,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/daa9af6937aa432ba431727e1bebdc8f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/daa9af6937aa432ba431727e1bebdc8f 2024-11-19T17:49:39,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/f88fa40958e343c7a95b6c7b57531f2c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/f88fa40958e343c7a95b6c7b57531f2c 2024-11-19T17:49:39,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/811f078f0ec34fd087a3f9924291cfa8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/811f078f0ec34fd087a3f9924291cfa8 2024-11-19T17:49:39,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/85121ff20bbf46a2aa80eddfb6990986 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/85121ff20bbf46a2aa80eddfb6990986 2024-11-19T17:49:39,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/044e0b9550134e3e9db2afc1afca0af0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/044e0b9550134e3e9db2afc1afca0af0 2024-11-19T17:49:39,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/adc8e33effab4f96a40af2a80ebef072 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/adc8e33effab4f96a40af2a80ebef072 2024-11-19T17:49:39,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c1e0344dabbe4edcb9fa82e287a4e6b4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/c1e0344dabbe4edcb9fa82e287a4e6b4 2024-11-19T17:49:39,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/1595c0a2e38b41648172266d44e36e74 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/1595c0a2e38b41648172266d44e36e74 2024-11-19T17:49:39,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/3de0df7ed2704d17b8cf89d14ce261b1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/3de0df7ed2704d17b8cf89d14ce261b1 2024-11-19T17:49:39,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/bc3447371b4c473984119d456a6b5d0d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/bc3447371b4c473984119d456a6b5d0d 2024-11-19T17:49:39,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0a6904df278b478788e7eab307e11b2a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/0a6904df278b478788e7eab307e11b2a 2024-11-19T17:49:39,806 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/recovered.edits/467.seqid, newMaxSeqId=467, maxSeqId=1 2024-11-19T17:49:39,806 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. 2024-11-19T17:49:39,806 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for f7aa8054d507e358fea7889f8e6a6f0a: Waiting for close lock at 1732038578465Running coprocessor pre-close hooks at 1732038578465Disabling compacts and flushes for region at 1732038578465Disabling writes for close at 1732038578466 (+1 ms)Obtaining lock to block concurrent updates at 1732038578466Preparing flush snapshotting stores in f7aa8054d507e358fea7889f8e6a6f0a at 1732038578466Finished memstore snapshotting TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732038578466Flushing stores of TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a. at 1732038578467 (+1 ms)Flushing f7aa8054d507e358fea7889f8e6a6f0a/A: creating writer at 1732038578467Flushing f7aa8054d507e358fea7889f8e6a6f0a/A: appending metadata at 1732038578472 (+5 ms)Flushing f7aa8054d507e358fea7889f8e6a6f0a/A: closing flushed file at 1732038578472Flushing f7aa8054d507e358fea7889f8e6a6f0a/B: creating writer at 1732038578887 (+415 ms)Flushing f7aa8054d507e358fea7889f8e6a6f0a/B: appending metadata at 1732038578890 (+3 ms)Flushing f7aa8054d507e358fea7889f8e6a6f0a/B: closing flushed file at 1732038578890Flushing f7aa8054d507e358fea7889f8e6a6f0a/C: creating writer at 1732038579298 (+408 ms)Flushing f7aa8054d507e358fea7889f8e6a6f0a/C: appending metadata at 1732038579300 (+2 ms)Flushing f7aa8054d507e358fea7889f8e6a6f0a/C: closing flushed file at 1732038579300Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77fac167: reopening flushed file at 1732038579708 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d24c138: reopening flushed file at 1732038579712 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@182e7a: reopening flushed file at 1732038579716 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f7aa8054d507e358fea7889f8e6a6f0a in 1254ms, sequenceid=464, compaction requested=true at 1732038579720 (+4 ms)Writing region close event to WAL at 1732038579803 (+83 ms)Running coprocessor post-close hooks at 1732038579806 (+3 ms)Closed at 1732038579806 2024-11-19T17:49:39,808 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:39,808 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=f7aa8054d507e358fea7889f8e6a6f0a, regionState=CLOSED 2024-11-19T17:49:39,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:49:39,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T17:49:39,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure f7aa8054d507e358fea7889f8e6a6f0a, server=26d3106abcfe,36839,1732038481522 in 1.4980 sec 2024-11-19T17:49:39,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-19T17:49:39,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7aa8054d507e358fea7889f8e6a6f0a, UNASSIGN in 1.5020 sec 2024-11-19T17:49:39,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T17:49:39,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5050 sec 2024-11-19T17:49:39,816 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038579816"}]},"ts":"1732038579816"} 2024-11-19T17:49:39,817 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:49:39,817 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:49:39,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5150 sec 2024-11-19T17:49:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:49:40,448 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:40,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:49:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:40,452 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T17:49:40,453 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:40,457 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:40,460 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/recovered.edits] 2024-11-19T17:49:40,466 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/41b1404c2a2e437fb609a4a203ca8944 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/41b1404c2a2e437fb609a4a203ca8944 2024-11-19T17:49:40,468 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f008d82c5bf41109c3c97e17ef7a281 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/5f008d82c5bf41109c3c97e17ef7a281 2024-11-19T17:49:40,470 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2cf576c779a42808b86cbb3edf4d93b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/d2cf576c779a42808b86cbb3edf4d93b 2024-11-19T17:49:40,472 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/db1c565ad1db435483576f622cb02ad0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/A/db1c565ad1db435483576f622cb02ad0 2024-11-19T17:49:40,475 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2a2f833fe1294d61b2c37052bedb0c8c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/2a2f833fe1294d61b2c37052bedb0c8c 2024-11-19T17:49:40,477 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/655f8b04693048c8a6d44b61c5302901 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/655f8b04693048c8a6d44b61c5302901 2024-11-19T17:49:40,479 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6ae0b2c1532843faa259d5eee8c767c2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/6ae0b2c1532843faa259d5eee8c767c2 2024-11-19T17:49:40,481 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f376dc002356443b9349ac28ff5aacde to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/B/f376dc002356443b9349ac28ff5aacde 2024-11-19T17:49:40,485 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/75840f34454f4b3ba7280d996475ac17 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/75840f34454f4b3ba7280d996475ac17 2024-11-19T17:49:40,487 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/83f2acd60b8947dcade6934359a40df7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/83f2acd60b8947dcade6934359a40df7 2024-11-19T17:49:40,489 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8e1b421644d04f8fbbfbe6a3a2efdbe6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/8e1b421644d04f8fbbfbe6a3a2efdbe6 2024-11-19T17:49:40,490 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/b68a0d2cc0834b1fa4730bea6d05dc9a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/C/b68a0d2cc0834b1fa4730bea6d05dc9a 2024-11-19T17:49:40,495 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/recovered.edits/467.seqid to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a/recovered.edits/467.seqid 2024-11-19T17:49:40,495 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/f7aa8054d507e358fea7889f8e6a6f0a 2024-11-19T17:49:40,495 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:49:40,498 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:40,503 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:49:40,506 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:49:40,508 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:40,508 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:49:40,508 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732038580508"}]},"ts":"9223372036854775807"} 2024-11-19T17:49:40,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:49:40,510 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f7aa8054d507e358fea7889f8e6a6f0a, NAME => 'TestAcidGuarantees,,1732038550238.f7aa8054d507e358fea7889f8e6a6f0a.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:49:40,510 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:49:40,510 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732038580510"}]},"ts":"9223372036854775807"} 2024-11-19T17:49:40,512 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:49:40,513 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:40,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-19T17:49:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T17:49:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:49:40,559 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:40,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 238), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=551 (was 568), ProcessCount=11 (was 11), AvailableMemoryMB=9498 (was 9611) 2024-11-19T17:49:40,573 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=551, ProcessCount=11, AvailableMemoryMB=9498 2024-11-19T17:49:40,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:49:40,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:49:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:40,577 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:49:40,577 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:40,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-19T17:49:40,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T17:49:40,578 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:49:40,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742184_1360 (size=959) 2024-11-19T17:49:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T17:49:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T17:49:40,975 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:49:40,986 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d 2024-11-19T17:49:40,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742185_1361 (size=53) 2024-11-19T17:49:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T17:49:41,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:49:41,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 47a3c46e7ce46ae36be91291b307632c, disabling compactions & flushes 2024-11-19T17:49:41,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:41,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:41,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. after waiting 0 ms 2024-11-19T17:49:41,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:41,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:41,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 47a3c46e7ce46ae36be91291b307632c: Waiting for close lock at 1732038581393Disabling compacts and flushes for region at 1732038581393Disabling writes for close at 1732038581393Writing region close event to WAL at 1732038581393Closed at 1732038581393 2024-11-19T17:49:41,394 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:49:41,394 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732038581394"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732038581394"}]},"ts":"1732038581394"} 2024-11-19T17:49:41,396 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:49:41,396 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:49:41,397 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038581396"}]},"ts":"1732038581396"} 2024-11-19T17:49:41,398 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:49:41,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, ASSIGN}] 2024-11-19T17:49:41,399 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, ASSIGN 2024-11-19T17:49:41,399 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, ASSIGN; state=OFFLINE, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=false 2024-11-19T17:49:41,551 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=47a3c46e7ce46ae36be91291b307632c, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:41,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, ASSIGN because future has completed 2024-11-19T17:49:41,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:49:41,714 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:41,714 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:49:41,715 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,715 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:49:41,715 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,715 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,717 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T17:49:41,720 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47a3c46e7ce46ae36be91291b307632c columnFamilyName A 2024-11-19T17:49:41,720 DEBUG [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:41,721 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(327): Store=47a3c46e7ce46ae36be91291b307632c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:41,721 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,723 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47a3c46e7ce46ae36be91291b307632c columnFamilyName B 2024-11-19T17:49:41,723 DEBUG [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:41,724 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(327): Store=47a3c46e7ce46ae36be91291b307632c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:41,724 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,725 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47a3c46e7ce46ae36be91291b307632c columnFamilyName C 2024-11-19T17:49:41,725 DEBUG [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:41,726 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(327): Store=47a3c46e7ce46ae36be91291b307632c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:41,726 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,727 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,728 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,729 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,729 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,730 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:49:41,732 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,734 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:49:41,735 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 47a3c46e7ce46ae36be91291b307632c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65559040, jitterRate=-0.02309417724609375}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:49:41,735 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:41,736 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 47a3c46e7ce46ae36be91291b307632c: Running coprocessor pre-open hook at 1732038581715Writing region info on filesystem at 1732038581715Initializing all the Stores at 1732038581717 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038581717Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038581717Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038581717Cleaning up temporary data from old regions at 1732038581729 (+12 ms)Running coprocessor post-open hooks at 1732038581735 (+6 ms)Region opened successfully at 1732038581736 (+1 ms) 2024-11-19T17:49:41,737 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., pid=95, masterSystemTime=1732038581709 2024-11-19T17:49:41,739 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:41,739 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:41,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=47a3c46e7ce46ae36be91291b307632c, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:41,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:49:41,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-19T17:49:41,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 in 187 msec 2024-11-19T17:49:41,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T17:49:41,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, ASSIGN in 346 msec 2024-11-19T17:49:41,747 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:49:41,747 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038581747"}]},"ts":"1732038581747"} 2024-11-19T17:49:41,749 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:49:41,750 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:49:41,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-19T17:49:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T17:49:42,729 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:42,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:42,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:42,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:42,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c83b01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:42,736 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:42,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c0fb5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:42,737 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:42,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:42,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:42,741 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:49:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,41481,1732038480627 2024-11-19T17:49:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f52a302 2024-11-19T17:49:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:49:42,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:49:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:49:42,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:49:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T17:49:42,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742186_1362 (size=995) 2024-11-19T17:49:43,160 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T17:49:43,160 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T17:49:43,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:49:43,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, REOPEN/MOVE}] 2024-11-19T17:49:43,168 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, REOPEN/MOVE 2024-11-19T17:49:43,169 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=47a3c46e7ce46ae36be91291b307632c, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, REOPEN/MOVE because future has completed 2024-11-19T17:49:43,170 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:49:43,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:49:43,323 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,323 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:49:43,323 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 47a3c46e7ce46ae36be91291b307632c, disabling compactions & flushes 2024-11-19T17:49:43,323 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,323 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,323 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. after waiting 0 ms 2024-11-19T17:49:43,323 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,331 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T17:49:43,332 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,332 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 47a3c46e7ce46ae36be91291b307632c: Waiting for close lock at 1732038583323Running coprocessor pre-close hooks at 1732038583323Disabling compacts and flushes for region at 1732038583323Disabling writes for close at 1732038583323Writing region close event to WAL at 1732038583324 (+1 ms)Running coprocessor post-close hooks at 1732038583331 (+7 ms)Closed at 1732038583331 2024-11-19T17:49:43,332 WARN [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 47a3c46e7ce46ae36be91291b307632c to self. 2024-11-19T17:49:43,334 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,334 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=47a3c46e7ce46ae36be91291b307632c, regionState=CLOSED 2024-11-19T17:49:43,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:49:43,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-19T17:49:43,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 in 167 msec 2024-11-19T17:49:43,340 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, REOPEN/MOVE; state=CLOSED, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=true 2024-11-19T17:49:43,491 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=47a3c46e7ce46ae36be91291b307632c, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, REOPEN/MOVE because future has completed 2024-11-19T17:49:43,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:49:43,649 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,649 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:49:43,649 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,649 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:49:43,649 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,649 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,650 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,651 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47a3c46e7ce46ae36be91291b307632c columnFamilyName A 2024-11-19T17:49:43,652 DEBUG [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:43,652 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(327): Store=47a3c46e7ce46ae36be91291b307632c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:43,653 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,653 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47a3c46e7ce46ae36be91291b307632c columnFamilyName B 2024-11-19T17:49:43,653 DEBUG [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:43,654 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(327): Store=47a3c46e7ce46ae36be91291b307632c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:43,654 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,655 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47a3c46e7ce46ae36be91291b307632c columnFamilyName C 2024-11-19T17:49:43,655 DEBUG [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:43,655 INFO [StoreOpener-47a3c46e7ce46ae36be91291b307632c-1 {}] regionserver.HStore(327): Store=47a3c46e7ce46ae36be91291b307632c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:49:43,655 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,656 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,656 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,657 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,657 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,658 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:49:43,659 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,660 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 47a3c46e7ce46ae36be91291b307632c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68536991, jitterRate=0.021280750632286072}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:49:43,660 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,660 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 47a3c46e7ce46ae36be91291b307632c: Running coprocessor pre-open hook at 1732038583649Writing region info on filesystem at 1732038583649Initializing all the Stores at 1732038583650 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038583650Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038583650Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038583650Cleaning up temporary data from old regions at 1732038583657 (+7 ms)Running coprocessor post-open hooks at 1732038583660 (+3 ms)Region opened successfully at 1732038583660 2024-11-19T17:49:43,661 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., pid=100, masterSystemTime=1732038583646 2024-11-19T17:49:43,663 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,663 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,663 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=47a3c46e7ce46ae36be91291b307632c, regionState=OPEN, openSeqNum=5, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:49:43,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-19T17:49:43,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 in 171 msec 2024-11-19T17:49:43,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T17:49:43,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, REOPEN/MOVE in 500 msec 2024-11-19T17:49:43,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-19T17:49:43,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 505 msec 2024-11-19T17:49:43,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-11-19T17:49:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-19T17:49:43,674 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:43,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,676 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240cebca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,677 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@165daf7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,679 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b1cb16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,682 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ca947f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,684 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34895bf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,686 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b62a1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,688 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19db36cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,689 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e893e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,692 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7283c019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,694 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e5ba27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,696 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ec5577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,698 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f20359f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,700 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582d624d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,702 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a62c0a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,707 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56edbe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a23a2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,711 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480c00de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,713 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53714996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:49:43,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:49:43,727 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:49:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:49:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:49:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e46617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:49:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:49:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:49:43,729 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:49:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450e042d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:49:43,730 DEBUG [Thread-1639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,730 DEBUG [Thread-1643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,730 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,731 DEBUG [Thread-1652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,731 DEBUG [Thread-1647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,732 DEBUG [Thread-1656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,732 DEBUG [Thread-1650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,733 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:43,733 DEBUG [Thread-1654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,734 DEBUG [Thread-1641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,735 DEBUG [Thread-1645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:49:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T17:49:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T17:49:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:49:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:49:43,742 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,744 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:43,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:49:43,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:49:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:43,753 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:43,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:49:43,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194be91cf408c1471db8d9ee5484ee2fed_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038583747/Put/seqid=0 2024-11-19T17:49:43,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742187_1363 (size=12154) 2024-11-19T17:49:43,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038643833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038643837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038643835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038643843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T17:49:43,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038643844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:43,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038643949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038643949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038643958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038643958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:43,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038643958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:44,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:44,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T17:49:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038644158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038644158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038644169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038644169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038644169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,194 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:44,197 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194be91cf408c1471db8d9ee5484ee2fed_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be91cf408c1471db8d9ee5484ee2fed_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:44,198 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/776eba253d594bd29f6c3a1e2b7e0597, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:44,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/776eba253d594bd29f6c3a1e2b7e0597 is 175, key is test_row_0/A:col10/1732038583747/Put/seqid=0 2024-11-19T17:49:44,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:44,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:44,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742188_1364 (size=30955) 2024-11-19T17:49:44,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:44,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:44,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T17:49:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038644468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038644468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038644479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038644479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038644479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:44,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:44,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,611 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/776eba253d594bd29f6c3a1e2b7e0597 2024-11-19T17:49:44,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/b1e7e76244634fe68d7e529fa2d7f2a6 is 50, key is test_row_0/B:col10/1732038583747/Put/seqid=0 2024-11-19T17:49:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742189_1365 (size=12001) 2024-11-19T17:49:44,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T17:49:44,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038644978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038644978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038644988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:44,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:44,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:44,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:44,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:44,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:45,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038644998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:45,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:45,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:45,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038644999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:45,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:45,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/b1e7e76244634fe68d7e529fa2d7f2a6 2024-11-19T17:49:45,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/b1e7e41d52a347ebb0dce2b392a13152 is 50, key is test_row_0/C:col10/1732038583747/Put/seqid=0 2024-11-19T17:49:45,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742190_1366 (size=12001) 2024-11-19T17:49:45,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/b1e7e41d52a347ebb0dce2b392a13152 2024-11-19T17:49:45,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/776eba253d594bd29f6c3a1e2b7e0597 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/776eba253d594bd29f6c3a1e2b7e0597 2024-11-19T17:49:45,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/776eba253d594bd29f6c3a1e2b7e0597, entries=150, sequenceid=15, filesize=30.2 K 2024-11-19T17:49:45,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/b1e7e76244634fe68d7e529fa2d7f2a6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b1e7e76244634fe68d7e529fa2d7f2a6 2024-11-19T17:49:45,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b1e7e76244634fe68d7e529fa2d7f2a6, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T17:49:45,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/b1e7e41d52a347ebb0dce2b392a13152 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/b1e7e41d52a347ebb0dce2b392a13152 2024-11-19T17:49:45,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/b1e7e41d52a347ebb0dce2b392a13152, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T17:49:45,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 47a3c46e7ce46ae36be91291b307632c in 1335ms, sequenceid=15, compaction requested=false 2024-11-19T17:49:45,088 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:49:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:45,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T17:49:45,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:45,151 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:49:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a563a89783444714ba2c0e63794703fc_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038583821/Put/seqid=0 2024-11-19T17:49:45,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742191_1367 (size=12154) 2024-11-19T17:49:45,331 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:49:45,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:45,569 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a563a89783444714ba2c0e63794703fc_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a563a89783444714ba2c0e63794703fc_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:45,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/05787550c7254a68b2ce1477f2e0cb4a, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:45,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/05787550c7254a68b2ce1477f2e0cb4a is 175, key is test_row_0/A:col10/1732038583821/Put/seqid=0 2024-11-19T17:49:45,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742192_1368 (size=30955) 2024-11-19T17:49:45,594 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/05787550c7254a68b2ce1477f2e0cb4a 2024-11-19T17:49:45,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/7dd714362d6243dab46f3d059ee807df is 50, key is test_row_0/B:col10/1732038583821/Put/seqid=0 2024-11-19T17:49:45,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742193_1369 (size=12001) 2024-11-19T17:49:45,622 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/7dd714362d6243dab46f3d059ee807df 2024-11-19T17:49:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/d8c311d6a8d442348dd778e9848b5925 is 50, key is test_row_0/C:col10/1732038583821/Put/seqid=0 2024-11-19T17:49:45,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742194_1370 (size=12001) 2024-11-19T17:49:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T17:49:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:46,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038646011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038646016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,046 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/d8c311d6a8d442348dd778e9848b5925 2024-11-19T17:49:46,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/05787550c7254a68b2ce1477f2e0cb4a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/05787550c7254a68b2ce1477f2e0cb4a 2024-11-19T17:49:46,052 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/05787550c7254a68b2ce1477f2e0cb4a, entries=150, sequenceid=40, filesize=30.2 K 2024-11-19T17:49:46,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/7dd714362d6243dab46f3d059ee807df as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7dd714362d6243dab46f3d059ee807df 2024-11-19T17:49:46,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038646043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038646043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038646043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,057 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7dd714362d6243dab46f3d059ee807df, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T17:49:46,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/d8c311d6a8d442348dd778e9848b5925 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d8c311d6a8d442348dd778e9848b5925 2024-11-19T17:49:46,060 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d8c311d6a8d442348dd778e9848b5925, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T17:49:46,061 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 47a3c46e7ce46ae36be91291b307632c in 910ms, sequenceid=40, compaction requested=false 2024-11-19T17:49:46,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:46,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:46,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T17:49:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T17:49:46,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T17:49:46,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3190 sec 2024-11-19T17:49:46,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3320 sec 2024-11-19T17:49:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:46,153 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:46,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:49:46,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c477037a15d041a8b171c68b66f88cba_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038586152/Put/seqid=0 2024-11-19T17:49:46,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742195_1371 (size=14594) 2024-11-19T17:49:46,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038646183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038646190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038646191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038646192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038646192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038646298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038646298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038646309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038646309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038646309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038646508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038646509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038646519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038646519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038646519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,566 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:46,569 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c477037a15d041a8b171c68b66f88cba_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c477037a15d041a8b171c68b66f88cba_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:46,570 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/4edb107b2a214c5eb1dda0470708efc4, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:46,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/4edb107b2a214c5eb1dda0470708efc4 is 175, key is test_row_0/A:col10/1732038586152/Put/seqid=0 2024-11-19T17:49:46,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742196_1372 (size=39549) 2024-11-19T17:49:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038646828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038646829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038646839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038646839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038646839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:46,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/4edb107b2a214c5eb1dda0470708efc4 2024-11-19T17:49:46,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c6e796dcf8e9469cbb2c75e7b249bd33 is 50, key is test_row_0/B:col10/1732038586152/Put/seqid=0 2024-11-19T17:49:46,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742197_1373 (size=12001) 2024-11-19T17:49:46,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c6e796dcf8e9469cbb2c75e7b249bd33 2024-11-19T17:49:47,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/3c06259f238d44a8a88d6e4c77dfe30b is 50, key is test_row_0/C:col10/1732038586152/Put/seqid=0 2024-11-19T17:49:47,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742198_1374 (size=12001) 2024-11-19T17:49:47,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038647338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:47,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038647338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:47,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038647348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:47,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038647349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038647359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:47,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/3c06259f238d44a8a88d6e4c77dfe30b 2024-11-19T17:49:47,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/4edb107b2a214c5eb1dda0470708efc4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/4edb107b2a214c5eb1dda0470708efc4 2024-11-19T17:49:47,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/4edb107b2a214c5eb1dda0470708efc4, entries=200, sequenceid=53, filesize=38.6 K 2024-11-19T17:49:47,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c6e796dcf8e9469cbb2c75e7b249bd33 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c6e796dcf8e9469cbb2c75e7b249bd33 2024-11-19T17:49:47,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c6e796dcf8e9469cbb2c75e7b249bd33, entries=150, sequenceid=53, filesize=11.7 K 2024-11-19T17:49:47,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/3c06259f238d44a8a88d6e4c77dfe30b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/3c06259f238d44a8a88d6e4c77dfe30b 2024-11-19T17:49:47,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/3c06259f238d44a8a88d6e4c77dfe30b, entries=150, sequenceid=53, filesize=11.7 K 2024-11-19T17:49:47,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47a3c46e7ce46ae36be91291b307632c in 1284ms, sequenceid=53, compaction requested=true 2024-11-19T17:49:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:47,437 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:47,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:47,438 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:47,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:47,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:47,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:47,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:47,439 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:47,439 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:47,439 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/A is initiating minor compaction (all files) 2024-11-19T17:49:47,439 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/B is initiating minor compaction (all files) 2024-11-19T17:49:47,439 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/A in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:47,439 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/776eba253d594bd29f6c3a1e2b7e0597, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/05787550c7254a68b2ce1477f2e0cb4a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/4edb107b2a214c5eb1dda0470708efc4] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=99.1 K 2024-11-19T17:49:47,439 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:47,439 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/776eba253d594bd29f6c3a1e2b7e0597, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/05787550c7254a68b2ce1477f2e0cb4a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/4edb107b2a214c5eb1dda0470708efc4] 2024-11-19T17:49:47,439 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/B in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:47,439 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b1e7e76244634fe68d7e529fa2d7f2a6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7dd714362d6243dab46f3d059ee807df, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c6e796dcf8e9469cbb2c75e7b249bd33] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=35.2 K 2024-11-19T17:49:47,443 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b1e7e76244634fe68d7e529fa2d7f2a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732038583742 2024-11-19T17:49:47,443 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 776eba253d594bd29f6c3a1e2b7e0597, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732038583742 2024-11-19T17:49:47,444 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd714362d6243dab46f3d059ee807df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038583821 2024-11-19T17:49:47,444 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05787550c7254a68b2ce1477f2e0cb4a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038583821 2024-11-19T17:49:47,444 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e796dcf8e9469cbb2c75e7b249bd33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732038586010 2024-11-19T17:49:47,444 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4edb107b2a214c5eb1dda0470708efc4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732038586010 2024-11-19T17:49:47,458 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:47,459 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/85e5c87b63fe4d6c8c1cf52dfc956a82 is 50, key is test_row_0/B:col10/1732038586152/Put/seqid=0 2024-11-19T17:49:47,461 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:47,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742199_1375 (size=12104) 2024-11-19T17:49:47,481 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111932f002f688d94fea983708a4f882b79a_47a3c46e7ce46ae36be91291b307632c store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:47,482 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/85e5c87b63fe4d6c8c1cf52dfc956a82 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/85e5c87b63fe4d6c8c1cf52dfc956a82 2024-11-19T17:49:47,483 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111932f002f688d94fea983708a4f882b79a_47a3c46e7ce46ae36be91291b307632c, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:47,483 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111932f002f688d94fea983708a4f882b79a_47a3c46e7ce46ae36be91291b307632c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:47,493 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/B of 47a3c46e7ce46ae36be91291b307632c into 85e5c87b63fe4d6c8c1cf52dfc956a82(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:47,493 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:47,493 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/B, priority=13, startTime=1732038587437; duration=0sec 2024-11-19T17:49:47,493 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:47,493 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:B 2024-11-19T17:49:47,493 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:47,495 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:47,495 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/C is initiating minor compaction (all files) 2024-11-19T17:49:47,495 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/C in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:47,495 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/b1e7e41d52a347ebb0dce2b392a13152, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d8c311d6a8d442348dd778e9848b5925, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/3c06259f238d44a8a88d6e4c77dfe30b] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=35.2 K 2024-11-19T17:49:47,495 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b1e7e41d52a347ebb0dce2b392a13152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732038583742 2024-11-19T17:49:47,495 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c311d6a8d442348dd778e9848b5925, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732038583821 2024-11-19T17:49:47,496 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c06259f238d44a8a88d6e4c77dfe30b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732038586010 2024-11-19T17:49:47,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742200_1376 (size=4469) 2024-11-19T17:49:47,518 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:47,519 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/ffdd03fa408e446f827b83fa299ef682 is 50, key is test_row_0/C:col10/1732038586152/Put/seqid=0 2024-11-19T17:49:47,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742201_1377 (size=12104) 2024-11-19T17:49:47,545 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/ffdd03fa408e446f827b83fa299ef682 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ffdd03fa408e446f827b83fa299ef682 2024-11-19T17:49:47,560 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/C of 47a3c46e7ce46ae36be91291b307632c into ffdd03fa408e446f827b83fa299ef682(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:47,560 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:47,560 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/C, priority=13, startTime=1732038587438; duration=0sec 2024-11-19T17:49:47,560 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:47,561 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:C 2024-11-19T17:49:47,914 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#A#compaction#322 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:47,915 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/325458c0b64a47eeb527093adee7e473 is 175, key is test_row_0/A:col10/1732038586152/Put/seqid=0 2024-11-19T17:49:47,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742202_1378 (size=31058) 2024-11-19T17:49:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T17:49:47,918 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:47,919 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T17:49:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T17:49:47,922 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:47,923 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:47,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T17:49:48,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T17:49:48,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:48,075 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T17:49:48,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198634ba8b3e184cf7bb17107290836fad_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038586190/Put/seqid=0 2024-11-19T17:49:48,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742203_1379 (size=12154) 2024-11-19T17:49:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T17:49:48,332 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/325458c0b64a47eeb527093adee7e473 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/325458c0b64a47eeb527093adee7e473 2024-11-19T17:49:48,340 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/A of 47a3c46e7ce46ae36be91291b307632c into 325458c0b64a47eeb527093adee7e473(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:48,340 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:48,340 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/A, priority=13, startTime=1732038587437; duration=0sec 2024-11-19T17:49:48,340 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:48,341 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:A 2024-11-19T17:49:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:48,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038648369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,382 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038648381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038648381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038648382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038648382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:48,489 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198634ba8b3e184cf7bb17107290836fad_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198634ba8b3e184cf7bb17107290836fad_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:48,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/d8fefd944bc14af9836ca6e86796a6e3, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:48,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/d8fefd944bc14af9836ca6e86796a6e3 is 175, key is test_row_0/A:col10/1732038586190/Put/seqid=0 2024-11-19T17:49:48,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038648498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038648499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038648499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038648499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742204_1380 (size=30955) 2024-11-19T17:49:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T17:49:48,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038648709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038648709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038648709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038648709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:48,906 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/d8fefd944bc14af9836ca6e86796a6e3 2024-11-19T17:49:48,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c2d1fe7a85a141058827b28f2f627092 is 50, key is test_row_0/B:col10/1732038586190/Put/seqid=0 2024-11-19T17:49:48,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742205_1381 (size=12001) 2024-11-19T17:49:48,920 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c2d1fe7a85a141058827b28f2f627092 2024-11-19T17:49:48,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/03576248eb30461f80119009056471c6 is 50, key is test_row_0/C:col10/1732038586190/Put/seqid=0 2024-11-19T17:49:48,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742206_1382 (size=12001) 2024-11-19T17:49:49,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038649028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038649028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038649028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038649028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T17:49:49,337 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/03576248eb30461f80119009056471c6 2024-11-19T17:49:49,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/d8fefd944bc14af9836ca6e86796a6e3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/d8fefd944bc14af9836ca6e86796a6e3 2024-11-19T17:49:49,344 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/d8fefd944bc14af9836ca6e86796a6e3, entries=150, sequenceid=78, filesize=30.2 K 2024-11-19T17:49:49,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c2d1fe7a85a141058827b28f2f627092 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d1fe7a85a141058827b28f2f627092 2024-11-19T17:49:49,348 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d1fe7a85a141058827b28f2f627092, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T17:49:49,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/03576248eb30461f80119009056471c6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/03576248eb30461f80119009056471c6 2024-11-19T17:49:49,351 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/03576248eb30461f80119009056471c6, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T17:49:49,352 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 47a3c46e7ce46ae36be91291b307632c in 1277ms, sequenceid=78, compaction requested=false 2024-11-19T17:49:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T17:49:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T17:49:49,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T17:49:49,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-19T17:49:49,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4370 sec 2024-11-19T17:49:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:49,544 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:49,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T17:49:49,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e87dc538af8742d9a4672134c4aa8c28_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038589541/Put/seqid=0 2024-11-19T17:49:49,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742207_1383 (size=14594) 2024-11-19T17:49:49,576 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:49,579 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e87dc538af8742d9a4672134c4aa8c28_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e87dc538af8742d9a4672134c4aa8c28_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:49,580 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/b66f4c3934204d99895ca5dfd25b9a83, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:49,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/b66f4c3934204d99895ca5dfd25b9a83 is 175, key is test_row_0/A:col10/1732038589541/Put/seqid=0 2024-11-19T17:49:49,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742208_1384 (size=39549) 2024-11-19T17:49:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038649595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038649601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038649602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038649603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038649709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038649719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038649719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038649719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038649918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038649929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038649939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038649939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,003 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/b66f4c3934204d99895ca5dfd25b9a83 2024-11-19T17:49:50,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/24ec8f2c957b4b5ca2c7f77aaaa732d8 is 50, key is test_row_0/B:col10/1732038589541/Put/seqid=0 2024-11-19T17:49:50,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742209_1385 (size=12001) 2024-11-19T17:49:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T17:49:50,078 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:50,079 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T17:49:50,081 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:49:50,081 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:50,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:49:50,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:49:50,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:50,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038650239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038650239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038650248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038650248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:49:50,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:50,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038650390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,391 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:49:50,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/24ec8f2c957b4b5ca2c7f77aaaa732d8 2024-11-19T17:49:50,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/9d60fab0f96547d9944cd279a880c27d is 50, key is test_row_0/C:col10/1732038589541/Put/seqid=0 2024-11-19T17:49:50,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742210_1386 (size=12001) 2024-11-19T17:49:50,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:49:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:49:50,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:50,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:49:50,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038650758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038650758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038650769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038650769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:50,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/9d60fab0f96547d9944cd279a880c27d 2024-11-19T17:49:50,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/b66f4c3934204d99895ca5dfd25b9a83 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b66f4c3934204d99895ca5dfd25b9a83 2024-11-19T17:49:50,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b66f4c3934204d99895ca5dfd25b9a83, entries=200, sequenceid=93, filesize=38.6 K 2024-11-19T17:49:50,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/24ec8f2c957b4b5ca2c7f77aaaa732d8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/24ec8f2c957b4b5ca2c7f77aaaa732d8 2024-11-19T17:49:50,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/24ec8f2c957b4b5ca2c7f77aaaa732d8, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T17:49:50,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/9d60fab0f96547d9944cd279a880c27d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9d60fab0f96547d9944cd279a880c27d 2024-11-19T17:49:50,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9d60fab0f96547d9944cd279a880c27d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T17:49:50,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 47a3c46e7ce46ae36be91291b307632c in 1295ms, sequenceid=93, compaction requested=true 2024-11-19T17:49:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:50,839 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:49:50,839 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:50,840 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:50,840 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:50,840 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/A is initiating minor compaction (all files) 2024-11-19T17:49:50,840 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/B is initiating minor compaction (all files) 2024-11-19T17:49:50,840 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/B in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,840 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/A in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,840 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/85e5c87b63fe4d6c8c1cf52dfc956a82, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d1fe7a85a141058827b28f2f627092, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/24ec8f2c957b4b5ca2c7f77aaaa732d8] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=35.3 K 2024-11-19T17:49:50,840 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/325458c0b64a47eeb527093adee7e473, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/d8fefd944bc14af9836ca6e86796a6e3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b66f4c3934204d99895ca5dfd25b9a83] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=99.2 K 2024-11-19T17:49:50,840 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,840 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/325458c0b64a47eeb527093adee7e473, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/d8fefd944bc14af9836ca6e86796a6e3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b66f4c3934204d99895ca5dfd25b9a83] 2024-11-19T17:49:50,841 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e5c87b63fe4d6c8c1cf52dfc956a82, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732038586010 2024-11-19T17:49:50,841 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325458c0b64a47eeb527093adee7e473, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732038586010 2024-11-19T17:49:50,841 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c2d1fe7a85a141058827b28f2f627092, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732038586181 2024-11-19T17:49:50,841 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8fefd944bc14af9836ca6e86796a6e3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732038586181 2024-11-19T17:49:50,841 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b66f4c3934204d99895ca5dfd25b9a83, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732038588378 2024-11-19T17:49:50,841 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ec8f2c957b4b5ca2c7f77aaaa732d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732038588380 2024-11-19T17:49:50,848 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:50,849 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:50,850 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/00b46f0ce3344805820b4d4e9967c15c is 50, key is test_row_0/B:col10/1732038589541/Put/seqid=0 2024-11-19T17:49:50,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:49:50,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,852 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T17:49:50,855 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119999d54af3699470a94cb880039677196_47a3c46e7ce46ae36be91291b307632c store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:50,857 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119999d54af3699470a94cb880039677196_47a3c46e7ce46ae36be91291b307632c, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:50,857 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119999d54af3699470a94cb880039677196_47a3c46e7ce46ae36be91291b307632c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:50,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742211_1387 (size=12207) 2024-11-19T17:49:50,890 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/00b46f0ce3344805820b4d4e9967c15c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/00b46f0ce3344805820b4d4e9967c15c 2024-11-19T17:49:50,899 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/B of 47a3c46e7ce46ae36be91291b307632c into 00b46f0ce3344805820b4d4e9967c15c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:50,899 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:50,899 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/B, priority=13, startTime=1732038590839; duration=0sec 2024-11-19T17:49:50,899 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:50,899 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:B 2024-11-19T17:49:50,899 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:50,900 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:50,900 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/C is initiating minor compaction (all files) 2024-11-19T17:49:50,900 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/C in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:50,901 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ffdd03fa408e446f827b83fa299ef682, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/03576248eb30461f80119009056471c6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9d60fab0f96547d9944cd279a880c27d] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=35.3 K 2024-11-19T17:49:50,901 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdd03fa408e446f827b83fa299ef682, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732038586010 2024-11-19T17:49:50,901 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 03576248eb30461f80119009056471c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732038586181 2024-11-19T17:49:50,901 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d60fab0f96547d9944cd279a880c27d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732038588380 2024-11-19T17:49:50,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f1d9b0f45a434db29d6db67bc2060e5d_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038589599/Put/seqid=0 2024-11-19T17:49:50,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742212_1388 (size=4469) 2024-11-19T17:49:50,909 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#A#compaction#331 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:50,909 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/b71e3b8654164a96a83d7303fe808985 is 175, key is test_row_0/A:col10/1732038589541/Put/seqid=0 2024-11-19T17:49:50,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742213_1389 (size=12154) 2024-11-19T17:49:50,914 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:50,914 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/75a903a2f1be4505879c15ceb093098c is 50, key is test_row_0/C:col10/1732038589541/Put/seqid=0 2024-11-19T17:49:50,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:50,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742214_1390 (size=31161) 2024-11-19T17:49:50,922 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f1d9b0f45a434db29d6db67bc2060e5d_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f1d9b0f45a434db29d6db67bc2060e5d_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/fcd3ba8a02804302bc21390d3dccdc49, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/fcd3ba8a02804302bc21390d3dccdc49 is 175, key is test_row_0/A:col10/1732038589599/Put/seqid=0 2024-11-19T17:49:50,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742215_1391 (size=12207) 2024-11-19T17:49:50,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742216_1392 (size=30955) 2024-11-19T17:49:50,975 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:49:50,975 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T17:49:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:49:51,326 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/b71e3b8654164a96a83d7303fe808985 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b71e3b8654164a96a83d7303fe808985 2024-11-19T17:49:51,329 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/A of 47a3c46e7ce46ae36be91291b307632c into b71e3b8654164a96a83d7303fe808985(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:51,329 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:51,329 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/A, priority=13, startTime=1732038590839; duration=0sec 2024-11-19T17:49:51,329 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:51,329 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:A 2024-11-19T17:49:51,343 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/75a903a2f1be4505879c15ceb093098c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/75a903a2f1be4505879c15ceb093098c 2024-11-19T17:49:51,347 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/C of 47a3c46e7ce46ae36be91291b307632c into 75a903a2f1be4505879c15ceb093098c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:51,347 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:51,347 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/C, priority=13, startTime=1732038590839; duration=0sec 2024-11-19T17:49:51,347 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:51,347 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:C 2024-11-19T17:49:51,350 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/fcd3ba8a02804302bc21390d3dccdc49 2024-11-19T17:49:51,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/ed6561b74a0c4be1ad628835f0898c3c is 50, key is test_row_0/B:col10/1732038589599/Put/seqid=0 2024-11-19T17:49:51,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742217_1393 (size=12001) 2024-11-19T17:49:51,759 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/ed6561b74a0c4be1ad628835f0898c3c 2024-11-19T17:49:51,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/a6262caf707a4cc3a371c47fcf71d077 is 50, key is test_row_0/C:col10/1732038589599/Put/seqid=0 2024-11-19T17:49:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:51,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:51,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742218_1394 (size=12001) 2024-11-19T17:49:51,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038651796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:51,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038651801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:51,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038651802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038651802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:51,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038651909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:51,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038651909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:51,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038651918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:51,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038651918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038652118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038652118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038652128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038652128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,174 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/a6262caf707a4cc3a371c47fcf71d077 2024-11-19T17:49:52,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/fcd3ba8a02804302bc21390d3dccdc49 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fcd3ba8a02804302bc21390d3dccdc49 2024-11-19T17:49:52,180 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fcd3ba8a02804302bc21390d3dccdc49, entries=150, sequenceid=115, filesize=30.2 K 2024-11-19T17:49:52,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/ed6561b74a0c4be1ad628835f0898c3c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ed6561b74a0c4be1ad628835f0898c3c 2024-11-19T17:49:52,184 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ed6561b74a0c4be1ad628835f0898c3c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T17:49:52,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/a6262caf707a4cc3a371c47fcf71d077 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/a6262caf707a4cc3a371c47fcf71d077 2024-11-19T17:49:52,187 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/a6262caf707a4cc3a371c47fcf71d077, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T17:49:52,188 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 47a3c46e7ce46ae36be91291b307632c in 1336ms, sequenceid=115, compaction requested=false 2024-11-19T17:49:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:52,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T17:49:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T17:49:52,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T17:49:52,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1080 sec 2024-11-19T17:49:52,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1120 sec 2024-11-19T17:49:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:49:52,229 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:52,229 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-19T17:49:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:49:52,230 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:52,231 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:52,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:49:52,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T17:49:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:52,383 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T17:49:52,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111965961b3478cc423380988001ee7b94c0_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038591800/Put/seqid=0 2024-11-19T17:49:52,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742219_1395 (size=12254) 2024-11-19T17:49:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:52,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038652468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038652477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038652480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038652481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:49:52,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038652589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038652589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038652589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038652600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038652798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,804 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111965961b3478cc423380988001ee7b94c0_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965961b3478cc423380988001ee7b94c0_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038652798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038652798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/7d8fc3a1210949b1bfc21a481e8c3dc8, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:52,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/7d8fc3a1210949b1bfc21a481e8c3dc8 is 175, key is test_row_0/A:col10/1732038591800/Put/seqid=0 2024-11-19T17:49:52,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742220_1396 (size=31055) 2024-11-19T17:49:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038652819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:49:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038653108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038653108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038653108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038653129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:53,209 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/7d8fc3a1210949b1bfc21a481e8c3dc8 2024-11-19T17:49:53,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/4c3cc264580c40ae9b2958f53f1274ef is 50, key is test_row_0/B:col10/1732038591800/Put/seqid=0 2024-11-19T17:49:53,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742221_1397 (size=12101) 2024-11-19T17:49:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:49:53,619 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/4c3cc264580c40ae9b2958f53f1274ef 2024-11-19T17:49:53,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038653618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038653618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:53,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038653618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/d9e7dbc29309474b8dc4f0e448df6078 is 50, key is test_row_0/C:col10/1732038591800/Put/seqid=0 2024-11-19T17:49:53,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742222_1398 (size=12101) 2024-11-19T17:49:53,640 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/d9e7dbc29309474b8dc4f0e448df6078 2024-11-19T17:49:53,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038653639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:53,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/7d8fc3a1210949b1bfc21a481e8c3dc8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/7d8fc3a1210949b1bfc21a481e8c3dc8 2024-11-19T17:49:53,657 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/7d8fc3a1210949b1bfc21a481e8c3dc8, entries=150, sequenceid=133, filesize=30.3 K 2024-11-19T17:49:53,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/4c3cc264580c40ae9b2958f53f1274ef as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4c3cc264580c40ae9b2958f53f1274ef 2024-11-19T17:49:53,672 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4c3cc264580c40ae9b2958f53f1274ef, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:49:53,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/d9e7dbc29309474b8dc4f0e448df6078 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d9e7dbc29309474b8dc4f0e448df6078 2024-11-19T17:49:53,684 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d9e7dbc29309474b8dc4f0e448df6078, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:49:53,687 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 47a3c46e7ce46ae36be91291b307632c in 1304ms, sequenceid=133, compaction requested=true 2024-11-19T17:49:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-19T17:49:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-19T17:49:53,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T17:49:53,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-19T17:49:53,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4640 sec 2024-11-19T17:49:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:49:54,379 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:54,379 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-19T17:49:54,381 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:49:54,382 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:54,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:54,422 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:54,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T17:49:54,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198338aa1dc25b40cd94984c1cabdff4e6_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038592449/Put/seqid=0 2024-11-19T17:49:54,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742223_1399 (size=14794) 2024-11-19T17:49:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:49:54,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038654487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:54,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T17:49:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038654598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038654639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038654639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038654639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:54,645 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:54,645 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:54,646 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:54,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038654648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:54,656 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:54,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T17:49:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:49:54,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038654819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:54,832 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:54,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T17:49:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:54,840 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198338aa1dc25b40cd94984c1cabdff4e6_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198338aa1dc25b40cd94984c1cabdff4e6_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:54,842 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/a4ac6ab308c54944b0fb12cb81fd3010, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:54,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/a4ac6ab308c54944b0fb12cb81fd3010 is 175, key is test_row_0/A:col10/1732038592449/Put/seqid=0 2024-11-19T17:49:54,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742224_1400 (size=39749) 2024-11-19T17:49:54,855 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/a4ac6ab308c54944b0fb12cb81fd3010 2024-11-19T17:49:54,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/ec69bb1f5c6847569d14fbdbb2c079ec is 50, key is test_row_0/B:col10/1732038592449/Put/seqid=0 2024-11-19T17:49:54,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742225_1401 (size=12151) 2024-11-19T17:49:54,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/ec69bb1f5c6847569d14fbdbb2c079ec 2024-11-19T17:49:54,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/cd7daf9b4bf1473cbbac4c1c92fbd70c is 50, key is test_row_0/C:col10/1732038592449/Put/seqid=0 2024-11-19T17:49:54,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742226_1402 (size=12151) 2024-11-19T17:49:54,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/cd7daf9b4bf1473cbbac4c1c92fbd70c 2024-11-19T17:49:54,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/a4ac6ab308c54944b0fb12cb81fd3010 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/a4ac6ab308c54944b0fb12cb81fd3010 2024-11-19T17:49:54,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/a4ac6ab308c54944b0fb12cb81fd3010, entries=200, sequenceid=155, filesize=38.8 K 2024-11-19T17:49:54,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/ec69bb1f5c6847569d14fbdbb2c079ec as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ec69bb1f5c6847569d14fbdbb2c079ec 2024-11-19T17:49:54,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ec69bb1f5c6847569d14fbdbb2c079ec, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T17:49:54,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/cd7daf9b4bf1473cbbac4c1c92fbd70c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cd7daf9b4bf1473cbbac4c1c92fbd70c 2024-11-19T17:49:54,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cd7daf9b4bf1473cbbac4c1c92fbd70c, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T17:49:54,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 47a3c46e7ce46ae36be91291b307632c in 490ms, sequenceid=155, compaction requested=true 2024-11-19T17:49:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:54,913 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:49:54,913 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:49:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:54,914 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:49:54,914 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:49:54,914 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/A is initiating minor compaction (all files) 2024-11-19T17:49:54,914 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/B is initiating minor compaction (all files) 2024-11-19T17:49:54,914 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/A in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,914 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/B in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,915 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b71e3b8654164a96a83d7303fe808985, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fcd3ba8a02804302bc21390d3dccdc49, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/7d8fc3a1210949b1bfc21a481e8c3dc8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/a4ac6ab308c54944b0fb12cb81fd3010] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=129.8 K 2024-11-19T17:49:54,915 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/00b46f0ce3344805820b4d4e9967c15c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ed6561b74a0c4be1ad628835f0898c3c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4c3cc264580c40ae9b2958f53f1274ef, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ec69bb1f5c6847569d14fbdbb2c079ec] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=47.3 K 2024-11-19T17:49:54,915 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,915 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b71e3b8654164a96a83d7303fe808985, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fcd3ba8a02804302bc21390d3dccdc49, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/7d8fc3a1210949b1bfc21a481e8c3dc8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/a4ac6ab308c54944b0fb12cb81fd3010] 2024-11-19T17:49:54,915 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b71e3b8654164a96a83d7303fe808985, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732038588380 2024-11-19T17:49:54,915 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b46f0ce3344805820b4d4e9967c15c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732038588380 2024-11-19T17:49:54,915 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcd3ba8a02804302bc21390d3dccdc49, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038589599 2024-11-19T17:49:54,915 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ed6561b74a0c4be1ad628835f0898c3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038589599 2024-11-19T17:49:54,915 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d8fc3a1210949b1bfc21a481e8c3dc8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038591786 2024-11-19T17:49:54,915 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c3cc264580c40ae9b2958f53f1274ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038591786 2024-11-19T17:49:54,916 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ac6ab308c54944b0fb12cb81fd3010, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038592449 2024-11-19T17:49:54,916 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ec69bb1f5c6847569d14fbdbb2c079ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038592449 2024-11-19T17:49:54,921 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:54,926 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#B#compaction#343 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:54,926 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/b5017abf812845198674256b41196571 is 50, key is test_row_0/B:col10/1732038592449/Put/seqid=0 2024-11-19T17:49:54,934 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411192b31668beba940e483825910094c11a2_47a3c46e7ce46ae36be91291b307632c store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:54,936 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411192b31668beba940e483825910094c11a2_47a3c46e7ce46ae36be91291b307632c, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:54,936 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192b31668beba940e483825910094c11a2_47a3c46e7ce46ae36be91291b307632c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:54,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742227_1403 (size=12493) 2024-11-19T17:49:54,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742228_1404 (size=4469) 2024-11-19T17:49:54,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T17:49:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:54,992 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T17:49:55,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e799b468ae6b4a0b85706ea3fd5cf873_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038594486/Put/seqid=0 2024-11-19T17:49:55,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742229_1405 (size=12304) 2024-11-19T17:49:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:49:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:55,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:55,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038655296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:55,348 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#A#compaction#342 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:55,349 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/b1b13ce64e0b4e35a71b4d64460d8957 is 175, key is test_row_0/A:col10/1732038592449/Put/seqid=0 2024-11-19T17:49:55,355 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/b5017abf812845198674256b41196571 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b5017abf812845198674256b41196571 2024-11-19T17:49:55,361 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/B of 47a3c46e7ce46ae36be91291b307632c into b5017abf812845198674256b41196571(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:55,361 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:55,361 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/B, priority=12, startTime=1732038594913; duration=0sec 2024-11-19T17:49:55,361 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:55,361 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:B 2024-11-19T17:49:55,361 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:49:55,362 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:49:55,362 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/C is initiating minor compaction (all files) 2024-11-19T17:49:55,362 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/C in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:55,362 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/75a903a2f1be4505879c15ceb093098c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/a6262caf707a4cc3a371c47fcf71d077, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d9e7dbc29309474b8dc4f0e448df6078, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cd7daf9b4bf1473cbbac4c1c92fbd70c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=47.3 K 2024-11-19T17:49:55,362 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 75a903a2f1be4505879c15ceb093098c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732038588380 2024-11-19T17:49:55,363 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a6262caf707a4cc3a371c47fcf71d077, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038589599 2024-11-19T17:49:55,363 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d9e7dbc29309474b8dc4f0e448df6078, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038591786 2024-11-19T17:49:55,364 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting cd7daf9b4bf1473cbbac4c1c92fbd70c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038592449 2024-11-19T17:49:55,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742230_1406 (size=31447) 2024-11-19T17:49:55,374 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/b1b13ce64e0b4e35a71b4d64460d8957 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b1b13ce64e0b4e35a71b4d64460d8957 2024-11-19T17:49:55,378 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/A of 47a3c46e7ce46ae36be91291b307632c into b1b13ce64e0b4e35a71b4d64460d8957(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:55,378 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:55,378 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:55,378 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/A, priority=12, startTime=1732038594913; duration=0sec 2024-11-19T17:49:55,378 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:55,378 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:A 2024-11-19T17:49:55,379 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/1cc4161b27a64fc48da92533b47769e0 is 50, key is test_row_0/C:col10/1732038592449/Put/seqid=0 2024-11-19T17:49:55,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742231_1407 (size=12493) 2024-11-19T17:49:55,389 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/1cc4161b27a64fc48da92533b47769e0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/1cc4161b27a64fc48da92533b47769e0 2024-11-19T17:49:55,394 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/C of 47a3c46e7ce46ae36be91291b307632c into 1cc4161b27a64fc48da92533b47769e0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:55,394 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:55,394 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/C, priority=12, startTime=1732038594913; duration=0sec 2024-11-19T17:49:55,394 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:55,394 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:C 2024-11-19T17:49:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:55,408 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e799b468ae6b4a0b85706ea3fd5cf873_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e799b468ae6b4a0b85706ea3fd5cf873_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:55,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/981cb88b9be349508b77ec4db53a3080, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:55,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/981cb88b9be349508b77ec4db53a3080 is 175, key is test_row_0/A:col10/1732038594486/Put/seqid=0 2024-11-19T17:49:55,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038655409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:55,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742232_1408 (size=31105) 2024-11-19T17:49:55,430 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/981cb88b9be349508b77ec4db53a3080 2024-11-19T17:49:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/8619ddca353f4031a1174944d75d2490 is 50, key is test_row_0/B:col10/1732038594486/Put/seqid=0 2024-11-19T17:49:55,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742233_1409 (size=12151) 2024-11-19T17:49:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:49:55,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038655619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:55,854 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/8619ddca353f4031a1174944d75d2490 2024-11-19T17:49:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/9de6ef3fc15f4415a01f6e7813ce2cd1 is 50, key is test_row_0/C:col10/1732038594486/Put/seqid=0 2024-11-19T17:49:55,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742234_1410 (size=12151) 2024-11-19T17:49:55,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038655929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:56,263 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/9de6ef3fc15f4415a01f6e7813ce2cd1 2024-11-19T17:49:56,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/981cb88b9be349508b77ec4db53a3080 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/981cb88b9be349508b77ec4db53a3080 2024-11-19T17:49:56,270 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/981cb88b9be349508b77ec4db53a3080, entries=150, sequenceid=169, filesize=30.4 K 2024-11-19T17:49:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/8619ddca353f4031a1174944d75d2490 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8619ddca353f4031a1174944d75d2490 2024-11-19T17:49:56,273 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8619ddca353f4031a1174944d75d2490, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T17:49:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/9de6ef3fc15f4415a01f6e7813ce2cd1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9de6ef3fc15f4415a01f6e7813ce2cd1 2024-11-19T17:49:56,277 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9de6ef3fc15f4415a01f6e7813ce2cd1, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T17:49:56,278 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 47a3c46e7ce46ae36be91291b307632c in 1286ms, sequenceid=169, compaction requested=false 2024-11-19T17:49:56,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:56,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:56,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-19T17:49:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-19T17:49:56,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T17:49:56,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-11-19T17:49:56,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.9020 sec 2024-11-19T17:49:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:56,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T17:49:56,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119306380d9819d4b018413ccc564776737_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038595246/Put/seqid=0 2024-11-19T17:49:56,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742235_1411 (size=14794) 2024-11-19T17:49:56,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038656503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:56,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:49:56,528 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:49:56,528 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:49:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-19T17:49:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:49:56,530 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:49:56,530 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:49:56,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:49:56,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038656609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:49:56,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038656658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:56,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038656658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:56,664 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,664 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038656669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038656669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:56,673 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,673 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:49:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:56,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:49:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:56,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:56,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:49:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:56,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038656828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:49:56,856 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:56,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119306380d9819d4b018413ccc564776737_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119306380d9819d4b018413ccc564776737_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:56,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/44ff32c7148c4f5eba32e9a6c1b86745, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:56,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/44ff32c7148c4f5eba32e9a6c1b86745 is 175, key is test_row_0/A:col10/1732038595246/Put/seqid=0 2024-11-19T17:49:56,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742236_1412 (size=39749) 2024-11-19T17:49:56,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:49:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:56,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:49:57,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:57,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038657149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:49:57,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/44ff32c7148c4f5eba32e9a6c1b86745 2024-11-19T17:49:57,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c2d9b5e93d154c8baf8361512ab73503 is 50, key is test_row_0/B:col10/1732038595246/Put/seqid=0 2024-11-19T17:49:57,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742237_1413 (size=12151) 2024-11-19T17:49:57,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c2d9b5e93d154c8baf8361512ab73503 2024-11-19T17:49:57,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/fdc4d515ef644dc89544efa2046369cd is 50, key is test_row_0/C:col10/1732038595246/Put/seqid=0 2024-11-19T17:49:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742238_1414 (size=12151) 2024-11-19T17:49:57,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:49:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:49:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:49:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:49:57,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038657659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:57,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:49:57,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/fdc4d515ef644dc89544efa2046369cd 2024-11-19T17:49:57,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/44ff32c7148c4f5eba32e9a6c1b86745 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/44ff32c7148c4f5eba32e9a6c1b86745 2024-11-19T17:49:57,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/44ff32c7148c4f5eba32e9a6c1b86745, entries=200, sequenceid=195, filesize=38.8 K 2024-11-19T17:49:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/c2d9b5e93d154c8baf8361512ab73503 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d9b5e93d154c8baf8361512ab73503 2024-11-19T17:49:57,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d9b5e93d154c8baf8361512ab73503, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T17:49:57,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/fdc4d515ef644dc89544efa2046369cd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/fdc4d515ef644dc89544efa2046369cd 2024-11-19T17:49:57,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/fdc4d515ef644dc89544efa2046369cd, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T17:49:57,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 47a3c46e7ce46ae36be91291b307632c in 1262ms, sequenceid=195, compaction requested=true 2024-11-19T17:49:57,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:57,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:49:57,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:57,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:49:57,710 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:57,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:57,710 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:57,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:49:57,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:57,710 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:57,710 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:57,710 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/B is initiating minor compaction (all files) 2024-11-19T17:49:57,710 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/A is initiating minor compaction (all files) 2024-11-19T17:49:57,711 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/B in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,711 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/A in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,711 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b5017abf812845198674256b41196571, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8619ddca353f4031a1174944d75d2490, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d9b5e93d154c8baf8361512ab73503] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=35.9 K 2024-11-19T17:49:57,711 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b1b13ce64e0b4e35a71b4d64460d8957, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/981cb88b9be349508b77ec4db53a3080, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/44ff32c7148c4f5eba32e9a6c1b86745] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=99.9 K 2024-11-19T17:49:57,711 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,711 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b1b13ce64e0b4e35a71b4d64460d8957, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/981cb88b9be349508b77ec4db53a3080, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/44ff32c7148c4f5eba32e9a6c1b86745] 2024-11-19T17:49:57,711 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b5017abf812845198674256b41196571, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038592449 2024-11-19T17:49:57,712 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8619ddca353f4031a1174944d75d2490, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732038594473 2024-11-19T17:49:57,712 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1b13ce64e0b4e35a71b4d64460d8957, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038592449 2024-11-19T17:49:57,712 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 981cb88b9be349508b77ec4db53a3080, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732038594473 2024-11-19T17:49:57,712 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c2d9b5e93d154c8baf8361512ab73503, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732038595246 2024-11-19T17:49:57,712 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ff32c7148c4f5eba32e9a6c1b86745, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732038595246 2024-11-19T17:49:57,718 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:57,719 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:57,719 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/8b39c8c71578443f89547eb2451bb8e7 is 50, key is test_row_0/B:col10/1732038595246/Put/seqid=0 2024-11-19T17:49:57,720 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119f98eee13ed1c4e9e8566f06fa35443e5_47a3c46e7ce46ae36be91291b307632c store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:57,722 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119f98eee13ed1c4e9e8566f06fa35443e5_47a3c46e7ce46ae36be91291b307632c, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:57,722 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f98eee13ed1c4e9e8566f06fa35443e5_47a3c46e7ce46ae36be91291b307632c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:57,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742239_1415 (size=12595) 2024-11-19T17:49:57,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742240_1416 (size=4469) 2024-11-19T17:49:57,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:49:57,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:57,753 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:49:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111939f9c2b9723b4cfeb586e906cff9d99a_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038596463/Put/seqid=0 2024-11-19T17:49:57,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742241_1417 (size=12304) 2024-11-19T17:49:58,129 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/8b39c8c71578443f89547eb2451bb8e7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8b39c8c71578443f89547eb2451bb8e7 2024-11-19T17:49:58,130 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:58,131 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/36150ccf20174877bbef69e2030c477d is 175, key is test_row_0/A:col10/1732038595246/Put/seqid=0 2024-11-19T17:49:58,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742242_1418 (size=31549) 2024-11-19T17:49:58,136 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/B of 47a3c46e7ce46ae36be91291b307632c into 8b39c8c71578443f89547eb2451bb8e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:58,136 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:58,136 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/B, priority=13, startTime=1732038597710; duration=0sec 2024-11-19T17:49:58,136 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:49:58,136 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:B 2024-11-19T17:49:58,136 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:49:58,137 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:49:58,138 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/C is initiating minor compaction (all files) 2024-11-19T17:49:58,138 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/C in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:58,138 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/1cc4161b27a64fc48da92533b47769e0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9de6ef3fc15f4415a01f6e7813ce2cd1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/fdc4d515ef644dc89544efa2046369cd] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=35.9 K 2024-11-19T17:49:58,138 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cc4161b27a64fc48da92533b47769e0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038592449 2024-11-19T17:49:58,141 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de6ef3fc15f4415a01f6e7813ce2cd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732038594473 2024-11-19T17:49:58,141 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc4d515ef644dc89544efa2046369cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732038595246 2024-11-19T17:49:58,142 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/36150ccf20174877bbef69e2030c477d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/36150ccf20174877bbef69e2030c477d 2024-11-19T17:49:58,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/A of 47a3c46e7ce46ae36be91291b307632c into 36150ccf20174877bbef69e2030c477d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:58,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:58,147 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/A, priority=13, startTime=1732038597710; duration=0sec 2024-11-19T17:49:58,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:58,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:A 2024-11-19T17:49:58,149 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:49:58,149 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/e696306753024f50974065cf46c6cfe2 is 50, key is test_row_0/C:col10/1732038595246/Put/seqid=0 2024-11-19T17:49:58,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742243_1419 (size=12595) 2024-11-19T17:49:58,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:58,166 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111939f9c2b9723b4cfeb586e906cff9d99a_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939f9c2b9723b4cfeb586e906cff9d99a_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/8200bd11b2b4426291d0fcfe406e8a0f, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/8200bd11b2b4426291d0fcfe406e8a0f is 175, key is test_row_0/A:col10/1732038596463/Put/seqid=0 2024-11-19T17:49:58,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742244_1420 (size=31105) 2024-11-19T17:49:58,557 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/e696306753024f50974065cf46c6cfe2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/e696306753024f50974065cf46c6cfe2 2024-11-19T17:49:58,560 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/C of 47a3c46e7ce46ae36be91291b307632c into e696306753024f50974065cf46c6cfe2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:49:58,560 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:58,560 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/C, priority=13, startTime=1732038597710; duration=0sec 2024-11-19T17:49:58,560 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:49:58,561 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:C 2024-11-19T17:49:58,575 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/8200bd11b2b4426291d0fcfe406e8a0f 2024-11-19T17:49:58,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/0526182ea9f84301aa152dd59956b820 is 50, key is test_row_0/B:col10/1732038596463/Put/seqid=0 2024-11-19T17:49:58,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742245_1421 (size=12151) 2024-11-19T17:49:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:49:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:49:58,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038658799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038658908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:58,992 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/0526182ea9f84301aa152dd59956b820 2024-11-19T17:49:59,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/ec17ed6f8320430b8d51c0badbff3ac1 is 50, key is test_row_0/C:col10/1732038596463/Put/seqid=0 2024-11-19T17:49:59,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742246_1422 (size=12151) 2024-11-19T17:49:59,039 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/ec17ed6f8320430b8d51c0badbff3ac1 2024-11-19T17:49:59,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/8200bd11b2b4426291d0fcfe406e8a0f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/8200bd11b2b4426291d0fcfe406e8a0f 2024-11-19T17:49:59,052 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/8200bd11b2b4426291d0fcfe406e8a0f, entries=150, sequenceid=208, filesize=30.4 K 2024-11-19T17:49:59,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/0526182ea9f84301aa152dd59956b820 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/0526182ea9f84301aa152dd59956b820 2024-11-19T17:49:59,061 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/0526182ea9f84301aa152dd59956b820, entries=150, sequenceid=208, filesize=11.9 K 2024-11-19T17:49:59,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/ec17ed6f8320430b8d51c0badbff3ac1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ec17ed6f8320430b8d51c0badbff3ac1 2024-11-19T17:49:59,067 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ec17ed6f8320430b8d51c0badbff3ac1, entries=150, sequenceid=208, filesize=11.9 K 2024-11-19T17:49:59,068 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47a3c46e7ce46ae36be91291b307632c in 1315ms, sequenceid=208, compaction requested=false 2024-11-19T17:49:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:49:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:49:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-19T17:49:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-19T17:49:59,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T17:49:59,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5390 sec 2024-11-19T17:49:59,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.5430 sec 2024-11-19T17:49:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:59,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:49:59,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:49:59,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a602aafbf1b344db890a3790910b5073_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038598798/Put/seqid=0 2024-11-19T17:49:59,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742247_1423 (size=14794) 2024-11-19T17:49:59,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038659181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038659288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:59,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038659498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:59,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:49:59,548 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a602aafbf1b344db890a3790910b5073_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a602aafbf1b344db890a3790910b5073_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:49:59,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/1b2730ae7046466b8e80a98c1acb742c, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:49:59,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/1b2730ae7046466b8e80a98c1acb742c is 175, key is test_row_0/A:col10/1732038598798/Put/seqid=0 2024-11-19T17:49:59,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742248_1424 (size=39749) 2024-11-19T17:49:59,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:49:59,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:49:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038659819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:49:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:49:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:49:59,966 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/1b2730ae7046466b8e80a98c1acb742c 2024-11-19T17:49:59,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/cfb70028a7c64653ad3b534cb170e7a6 is 50, key is test_row_0/B:col10/1732038598798/Put/seqid=0 2024-11-19T17:49:59,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742249_1425 (size=12151) 2024-11-19T17:50:00,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038660339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:00,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/cfb70028a7c64653ad3b534cb170e7a6 2024-11-19T17:50:00,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/652c0a33f3bb43d28c67f304140b25e0 is 50, key is test_row_0/C:col10/1732038598798/Put/seqid=0 2024-11-19T17:50:00,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742250_1426 (size=12151) 2024-11-19T17:50:00,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/652c0a33f3bb43d28c67f304140b25e0 2024-11-19T17:50:00,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/1b2730ae7046466b8e80a98c1acb742c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/1b2730ae7046466b8e80a98c1acb742c 2024-11-19T17:50:00,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/1b2730ae7046466b8e80a98c1acb742c, entries=200, sequenceid=235, filesize=38.8 K 2024-11-19T17:50:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/cfb70028a7c64653ad3b534cb170e7a6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cfb70028a7c64653ad3b534cb170e7a6 2024-11-19T17:50:00,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cfb70028a7c64653ad3b534cb170e7a6, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T17:50:00,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/652c0a33f3bb43d28c67f304140b25e0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/652c0a33f3bb43d28c67f304140b25e0 2024-11-19T17:50:00,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/652c0a33f3bb43d28c67f304140b25e0, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T17:50:00,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 47a3c46e7ce46ae36be91291b307632c in 1299ms, sequenceid=235, compaction requested=true 2024-11-19T17:50:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:00,420 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:50:00,420 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/A is initiating minor compaction (all files) 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/B is initiating minor compaction (all files) 2024-11-19T17:50:00,422 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/A in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:00,422 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/B in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:00,422 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/36150ccf20174877bbef69e2030c477d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/8200bd11b2b4426291d0fcfe406e8a0f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/1b2730ae7046466b8e80a98c1acb742c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=100.0 K 2024-11-19T17:50:00,422 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8b39c8c71578443f89547eb2451bb8e7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/0526182ea9f84301aa152dd59956b820, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cfb70028a7c64653ad3b534cb170e7a6] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=36.0 K 2024-11-19T17:50:00,422 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/36150ccf20174877bbef69e2030c477d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/8200bd11b2b4426291d0fcfe406e8a0f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/1b2730ae7046466b8e80a98c1acb742c] 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b39c8c71578443f89547eb2451bb8e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732038595246 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36150ccf20174877bbef69e2030c477d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732038595246 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8200bd11b2b4426291d0fcfe406e8a0f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732038596463 2024-11-19T17:50:00,422 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0526182ea9f84301aa152dd59956b820, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732038596463 2024-11-19T17:50:00,423 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb70028a7c64653ad3b534cb170e7a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732038598784 2024-11-19T17:50:00,423 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2730ae7046466b8e80a98c1acb742c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732038598784 2024-11-19T17:50:00,442 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:50:00,442 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/080accc7b72b457b8e54f3cadbd2923c is 50, key is test_row_0/B:col10/1732038598798/Put/seqid=0 2024-11-19T17:50:00,450 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:00,465 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194cdf9ea69bc54482bde6b7301c2f7cd3_47a3c46e7ce46ae36be91291b307632c store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:00,466 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194cdf9ea69bc54482bde6b7301c2f7cd3_47a3c46e7ce46ae36be91291b307632c, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:00,466 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194cdf9ea69bc54482bde6b7301c2f7cd3_47a3c46e7ce46ae36be91291b307632c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:00,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742251_1427 (size=12697) 2024-11-19T17:50:00,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742252_1428 (size=4469) 2024-11-19T17:50:00,493 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#A#compaction#361 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:50:00,493 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/fc47d9d6030d496cae900edfb6d3c670 is 175, key is test_row_0/A:col10/1732038598798/Put/seqid=0 2024-11-19T17:50:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742253_1429 (size=31651) 2024-11-19T17:50:00,514 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/fc47d9d6030d496cae900edfb6d3c670 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fc47d9d6030d496cae900edfb6d3c670 2024-11-19T17:50:00,532 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/A of 47a3c46e7ce46ae36be91291b307632c into fc47d9d6030d496cae900edfb6d3c670(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:00,532 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:00,532 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/A, priority=13, startTime=1732038600420; duration=0sec 2024-11-19T17:50:00,532 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:00,532 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:A 2024-11-19T17:50:00,532 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:00,533 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:00,533 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/C is initiating minor compaction (all files) 2024-11-19T17:50:00,533 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/C in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:00,533 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/e696306753024f50974065cf46c6cfe2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ec17ed6f8320430b8d51c0badbff3ac1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/652c0a33f3bb43d28c67f304140b25e0] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=36.0 K 2024-11-19T17:50:00,534 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting e696306753024f50974065cf46c6cfe2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732038595246 2024-11-19T17:50:00,535 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec17ed6f8320430b8d51c0badbff3ac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732038596463 2024-11-19T17:50:00,535 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 652c0a33f3bb43d28c67f304140b25e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732038598784 2024-11-19T17:50:00,542 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:50:00,542 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/9bb7ce1b193d4ce8905c14228b99b8e8 is 50, key is test_row_0/C:col10/1732038598798/Put/seqid=0 2024-11-19T17:50:00,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742254_1430 (size=12697) 2024-11-19T17:50:00,567 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/9bb7ce1b193d4ce8905c14228b99b8e8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9bb7ce1b193d4ce8905c14228b99b8e8 2024-11-19T17:50:00,572 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/C of 47a3c46e7ce46ae36be91291b307632c into 9bb7ce1b193d4ce8905c14228b99b8e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:00,572 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:00,572 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/C, priority=13, startTime=1732038600420; duration=0sec 2024-11-19T17:50:00,572 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:00,572 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:C 2024-11-19T17:50:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:50:00,699 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:00,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-19T17:50:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:50:00,701 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:00,702 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:00,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:00,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:00,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:50:00,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193fe33155743b4e32860d271ec9417be2_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038599179/Put/seqid=0 2024-11-19T17:50:00,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742255_1431 (size=14794) 2024-11-19T17:50:00,737 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:00,745 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193fe33155743b4e32860d271ec9417be2_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193fe33155743b4e32860d271ec9417be2_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:00,746 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/598cc11c03614ab4b2e7ab04193d01a7, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:00,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/598cc11c03614ab4b2e7ab04193d01a7 is 175, key is test_row_0/A:col10/1732038599179/Put/seqid=0 2024-11-19T17:50:00,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742256_1432 (size=39749) 2024-11-19T17:50:00,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038660754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038660764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038660762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038660765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:50:00,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:50:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:00,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:00,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038660868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:00,877 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/080accc7b72b457b8e54f3cadbd2923c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/080accc7b72b457b8e54f3cadbd2923c 2024-11-19T17:50:00,881 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/B of 47a3c46e7ce46ae36be91291b307632c into 080accc7b72b457b8e54f3cadbd2923c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:00,881 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:00,881 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/B, priority=13, startTime=1732038600420; duration=0sec 2024-11-19T17:50:00,881 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:00,881 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:B 2024-11-19T17:50:00,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038660879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038660879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:00,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038660879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:50:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:50:01,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038661079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038661098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038661098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038661098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,151 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/598cc11c03614ab4b2e7ab04193d01a7 2024-11-19T17:50:01,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/bfbe069ace284b69939d0e70d84fbbdc is 50, key is test_row_0/B:col10/1732038599179/Put/seqid=0 2024-11-19T17:50:01,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742257_1433 (size=12151) 2024-11-19T17:50:01,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/bfbe069ace284b69939d0e70d84fbbdc 2024-11-19T17:50:01,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:50:01,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/67c57439697047db9cf9f942fe596a7f is 50, key is test_row_0/C:col10/1732038599179/Put/seqid=0 2024-11-19T17:50:01,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742258_1434 (size=12151) 2024-11-19T17:50:01,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:50:01,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:01,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:50:01,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038661358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,361 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:01,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038661389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038661418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038661418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038661418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:50:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:01,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/67c57439697047db9cf9f942fe596a7f 2024-11-19T17:50:01,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/598cc11c03614ab4b2e7ab04193d01a7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/598cc11c03614ab4b2e7ab04193d01a7 2024-11-19T17:50:01,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/598cc11c03614ab4b2e7ab04193d01a7, entries=200, sequenceid=250, filesize=38.8 K 2024-11-19T17:50:01,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/bfbe069ace284b69939d0e70d84fbbdc as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/bfbe069ace284b69939d0e70d84fbbdc 2024-11-19T17:50:01,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/bfbe069ace284b69939d0e70d84fbbdc, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T17:50:01,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/67c57439697047db9cf9f942fe596a7f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/67c57439697047db9cf9f942fe596a7f 2024-11-19T17:50:01,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/67c57439697047db9cf9f942fe596a7f, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T17:50:01,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 47a3c46e7ce46ae36be91291b307632c in 907ms, sequenceid=250, compaction requested=false 2024-11-19T17:50:01,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:01,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:50:01,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:01,626 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:50:01,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c5393ee6bf844428b5d82f382740d86_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038600763/Put/seqid=0 2024-11-19T17:50:01,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742259_1435 (size=12454) 2024-11-19T17:50:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:01,660 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c5393ee6bf844428b5d82f382740d86_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c5393ee6bf844428b5d82f382740d86_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:01,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/53bbeacddae34c7aa02146e6698263a3, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:01,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/53bbeacddae34c7aa02146e6698263a3 is 175, key is test_row_0/A:col10/1732038600763/Put/seqid=0 2024-11-19T17:50:01,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742260_1436 (size=31255) 2024-11-19T17:50:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:50:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:01,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:01,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038661938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038661938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038661939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:01,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038661942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038662049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038662059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,111 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/53bbeacddae34c7aa02146e6698263a3 2024-11-19T17:50:02,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/d86670910ef24dd1b203de648854163b is 50, key is test_row_0/B:col10/1732038600763/Put/seqid=0 2024-11-19T17:50:02,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742261_1437 (size=12301) 2024-11-19T17:50:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038662258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038662269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,521 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/d86670910ef24dd1b203de648854163b 2024-11-19T17:50:02,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/88470a09b98547e48c8d242aec97e417 is 50, key is test_row_0/C:col10/1732038600763/Put/seqid=0 2024-11-19T17:50:02,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742262_1438 (size=12301) 2024-11-19T17:50:02,535 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/88470a09b98547e48c8d242aec97e417 2024-11-19T17:50:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/53bbeacddae34c7aa02146e6698263a3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/53bbeacddae34c7aa02146e6698263a3 2024-11-19T17:50:02,542 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/53bbeacddae34c7aa02146e6698263a3, entries=150, sequenceid=275, filesize=30.5 K 2024-11-19T17:50:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/d86670910ef24dd1b203de648854163b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/d86670910ef24dd1b203de648854163b 2024-11-19T17:50:02,546 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/d86670910ef24dd1b203de648854163b, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T17:50:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/88470a09b98547e48c8d242aec97e417 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/88470a09b98547e48c8d242aec97e417 2024-11-19T17:50:02,549 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/88470a09b98547e48c8d242aec97e417, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T17:50:02,549 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 47a3c46e7ce46ae36be91291b307632c in 923ms, sequenceid=275, compaction requested=true 2024-11-19T17:50:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-19T17:50:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-19T17:50:02,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T17:50:02,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8490 sec 2024-11-19T17:50:02,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8530 sec 2024-11-19T17:50:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:02,586 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:02,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:50:02,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190c3e04e8a4154ce18d906a1af82410ea_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038601919/Put/seqid=0 2024-11-19T17:50:02,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742263_1439 (size=14994) 2024-11-19T17:50:02,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038662652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038662659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038662769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038662769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:50:02,849 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:02,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-19T17:50:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:50:02,852 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:02,854 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:02,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:50:02,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36616 deadline: 1732038662958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,970 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36664 deadline: 1732038662969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,976 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038662979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038662979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:02,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:02,999 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190c3e04e8a4154ce18d906a1af82410ea_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190c3e04e8a4154ce18d906a1af82410ea_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:03,003 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/bfe2ccbc36e64a288ad1a027d92f4eef, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:03,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/bfe2ccbc36e64a288ad1a027d92f4eef is 175, key is test_row_0/A:col10/1732038601919/Put/seqid=0 2024-11-19T17:50:03,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742264_1440 (size=39949) 2024-11-19T17:50:03,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:50:03,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038663298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:03,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038663298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:03,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36626 deadline: 1732038663389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:03,390 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 47a3c46e7ce46ae36be91291b307632c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:03,426 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/bfe2ccbc36e64a288ad1a027d92f4eef 2024-11-19T17:50:03,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/7a04dc215dae45edb32ba8d6f0166986 is 50, key is test_row_0/B:col10/1732038601919/Put/seqid=0 2024-11-19T17:50:03,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742265_1441 (size=12301) 2024-11-19T17:50:03,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:03,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:03,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:50:03,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,736 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-11-19T17:50:03,736 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:03,736 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,736 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:03,742 INFO [Thread-1656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1656. 2024-11-19T17:50:03,742 DEBUG [Thread-1656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:03,742 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,742 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:03,744 INFO [Thread-1650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1650. 2024-11-19T17:50:03,744 DEBUG [Thread-1650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:03,744 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,744 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:03,746 INFO [Thread-1654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1654. 2024-11-19T17:50:03,746 DEBUG [Thread-1654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:03,746 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,746 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:03,755 INFO [Thread-1652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1652. 2024-11-19T17:50:03,755 DEBUG [Thread-1652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:03,755 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,755 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:03,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:03,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:03,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:03,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:03,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36674 deadline: 1732038663819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36682 deadline: 1732038663819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:03,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/7a04dc215dae45edb32ba8d6f0166986 2024-11-19T17:50:03,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/08d8bb6816a84146a8b945031b00931b is 50, key is test_row_0/C:col10/1732038601919/Put/seqid=0 2024-11-19T17:50:03,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742266_1442 (size=12301) 2024-11-19T17:50:03,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:03,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:50:04,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:04,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:04,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/08d8bb6816a84146a8b945031b00931b 2024-11-19T17:50:04,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/bfe2ccbc36e64a288ad1a027d92f4eef as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/bfe2ccbc36e64a288ad1a027d92f4eef 2024-11-19T17:50:04,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/bfe2ccbc36e64a288ad1a027d92f4eef, entries=200, sequenceid=288, filesize=39.0 K 2024-11-19T17:50:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/7a04dc215dae45edb32ba8d6f0166986 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7a04dc215dae45edb32ba8d6f0166986 2024-11-19T17:50:04,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7a04dc215dae45edb32ba8d6f0166986, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T17:50:04,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/08d8bb6816a84146a8b945031b00931b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/08d8bb6816a84146a8b945031b00931b 2024-11-19T17:50:04,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/08d8bb6816a84146a8b945031b00931b, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T17:50:04,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47a3c46e7ce46ae36be91291b307632c in 1686ms, sequenceid=288, compaction requested=true 2024-11-19T17:50:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:04,272 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:04,272 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47a3c46e7ce46ae36be91291b307632c:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:04,273 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142604 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:04,273 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:04,274 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/A is initiating minor compaction (all files) 2024-11-19T17:50:04,274 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/B is initiating minor compaction (all files) 2024-11-19T17:50:04,274 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/A in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,274 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/B in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,274 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fc47d9d6030d496cae900edfb6d3c670, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/598cc11c03614ab4b2e7ab04193d01a7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/53bbeacddae34c7aa02146e6698263a3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/bfe2ccbc36e64a288ad1a027d92f4eef] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=139.3 K 2024-11-19T17:50:04,274 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/080accc7b72b457b8e54f3cadbd2923c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/bfbe069ace284b69939d0e70d84fbbdc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/d86670910ef24dd1b203de648854163b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7a04dc215dae45edb32ba8d6f0166986] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=48.3 K 2024-11-19T17:50:04,274 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,274 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fc47d9d6030d496cae900edfb6d3c670, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/598cc11c03614ab4b2e7ab04193d01a7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/53bbeacddae34c7aa02146e6698263a3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/bfe2ccbc36e64a288ad1a027d92f4eef] 2024-11-19T17:50:04,274 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 080accc7b72b457b8e54f3cadbd2923c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732038598784 2024-11-19T17:50:04,274 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting fc47d9d6030d496cae900edfb6d3c670, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732038598784 2024-11-19T17:50:04,274 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 598cc11c03614ab4b2e7ab04193d01a7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732038599158 2024-11-19T17:50:04,275 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfbe069ace284b69939d0e70d84fbbdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732038599158 2024-11-19T17:50:04,275 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 53bbeacddae34c7aa02146e6698263a3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038600728 2024-11-19T17:50:04,275 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting d86670910ef24dd1b203de648854163b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038600728 2024-11-19T17:50:04,275 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe2ccbc36e64a288ad1a027d92f4eef, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038601919 2024-11-19T17:50:04,275 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a04dc215dae45edb32ba8d6f0166986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038601919 2024-11-19T17:50:04,282 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#B#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:50:04,282 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/b9ca6e7072c14ed49a65358a61e09be4 is 50, key is test_row_0/B:col10/1732038601919/Put/seqid=0 2024-11-19T17:50:04,283 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:04,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742267_1443 (size=12983) 2024-11-19T17:50:04,285 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a33852b2458f4af4b1c96f8d2ac2f8e6_47a3c46e7ce46ae36be91291b307632c store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:04,303 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a33852b2458f4af4b1c96f8d2ac2f8e6_47a3c46e7ce46ae36be91291b307632c, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:04,303 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a33852b2458f4af4b1c96f8d2ac2f8e6_47a3c46e7ce46ae36be91291b307632c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:04,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742268_1444 (size=4469) 2024-11-19T17:50:04,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:50:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,387 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T17:50:04,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194922130484034599b8494ddbf208c656_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038602648/Put/seqid=0 2024-11-19T17:50:04,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742269_1445 (size=12454) 2024-11-19T17:50:04,478 DEBUG [regionserver/26d3106abcfe:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-19T17:50:04,486 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/776eba253d594bd29f6c3a1e2b7e0597, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/05787550c7254a68b2ce1477f2e0cb4a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/4edb107b2a214c5eb1dda0470708efc4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/325458c0b64a47eeb527093adee7e473, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/d8fefd944bc14af9836ca6e86796a6e3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b66f4c3934204d99895ca5dfd25b9a83, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b71e3b8654164a96a83d7303fe808985, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fcd3ba8a02804302bc21390d3dccdc49, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/7d8fc3a1210949b1bfc21a481e8c3dc8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/a4ac6ab308c54944b0fb12cb81fd3010, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b1b13ce64e0b4e35a71b4d64460d8957, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/981cb88b9be349508b77ec4db53a3080, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/44ff32c7148c4f5eba32e9a6c1b86745, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/36150ccf20174877bbef69e2030c477d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/8200bd11b2b4426291d0fcfe406e8a0f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/1b2730ae7046466b8e80a98c1acb742c] to archive 2024-11-19T17:50:04,487 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:04,488 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/776eba253d594bd29f6c3a1e2b7e0597 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/776eba253d594bd29f6c3a1e2b7e0597 2024-11-19T17:50:04,490 WARN [regionserver/26d3106abcfe:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-19T17:50:04,490 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/05787550c7254a68b2ce1477f2e0cb4a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/05787550c7254a68b2ce1477f2e0cb4a 2024-11-19T17:50:04,491 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/4edb107b2a214c5eb1dda0470708efc4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/4edb107b2a214c5eb1dda0470708efc4 2024-11-19T17:50:04,492 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/325458c0b64a47eeb527093adee7e473 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/325458c0b64a47eeb527093adee7e473 2024-11-19T17:50:04,492 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/d8fefd944bc14af9836ca6e86796a6e3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/d8fefd944bc14af9836ca6e86796a6e3 2024-11-19T17:50:04,493 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b66f4c3934204d99895ca5dfd25b9a83 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b66f4c3934204d99895ca5dfd25b9a83 2024-11-19T17:50:04,494 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b71e3b8654164a96a83d7303fe808985 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b71e3b8654164a96a83d7303fe808985 2024-11-19T17:50:04,495 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fcd3ba8a02804302bc21390d3dccdc49 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fcd3ba8a02804302bc21390d3dccdc49 2024-11-19T17:50:04,495 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/7d8fc3a1210949b1bfc21a481e8c3dc8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/7d8fc3a1210949b1bfc21a481e8c3dc8 2024-11-19T17:50:04,496 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/a4ac6ab308c54944b0fb12cb81fd3010 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/a4ac6ab308c54944b0fb12cb81fd3010 2024-11-19T17:50:04,497 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b1b13ce64e0b4e35a71b4d64460d8957 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/b1b13ce64e0b4e35a71b4d64460d8957 2024-11-19T17:50:04,498 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/981cb88b9be349508b77ec4db53a3080 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/981cb88b9be349508b77ec4db53a3080 2024-11-19T17:50:04,499 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/44ff32c7148c4f5eba32e9a6c1b86745 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/44ff32c7148c4f5eba32e9a6c1b86745 2024-11-19T17:50:04,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/36150ccf20174877bbef69e2030c477d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/36150ccf20174877bbef69e2030c477d 2024-11-19T17:50:04,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/8200bd11b2b4426291d0fcfe406e8a0f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/8200bd11b2b4426291d0fcfe406e8a0f 2024-11-19T17:50:04,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/1b2730ae7046466b8e80a98c1acb742c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/1b2730ae7046466b8e80a98c1acb742c 2024-11-19T17:50:04,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b1e7e76244634fe68d7e529fa2d7f2a6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7dd714362d6243dab46f3d059ee807df, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/85e5c87b63fe4d6c8c1cf52dfc956a82, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c6e796dcf8e9469cbb2c75e7b249bd33, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d1fe7a85a141058827b28f2f627092, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/00b46f0ce3344805820b4d4e9967c15c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/24ec8f2c957b4b5ca2c7f77aaaa732d8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ed6561b74a0c4be1ad628835f0898c3c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4c3cc264580c40ae9b2958f53f1274ef, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b5017abf812845198674256b41196571, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ec69bb1f5c6847569d14fbdbb2c079ec, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8619ddca353f4031a1174944d75d2490, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8b39c8c71578443f89547eb2451bb8e7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d9b5e93d154c8baf8361512ab73503, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/0526182ea9f84301aa152dd59956b820, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cfb70028a7c64653ad3b534cb170e7a6] to archive 2024-11-19T17:50:04,504 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:04,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b1e7e76244634fe68d7e529fa2d7f2a6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b1e7e76244634fe68d7e529fa2d7f2a6 2024-11-19T17:50:04,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7dd714362d6243dab46f3d059ee807df to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7dd714362d6243dab46f3d059ee807df 2024-11-19T17:50:04,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/85e5c87b63fe4d6c8c1cf52dfc956a82 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/85e5c87b63fe4d6c8c1cf52dfc956a82 2024-11-19T17:50:04,508 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c6e796dcf8e9469cbb2c75e7b249bd33 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c6e796dcf8e9469cbb2c75e7b249bd33 2024-11-19T17:50:04,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d1fe7a85a141058827b28f2f627092 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d1fe7a85a141058827b28f2f627092 2024-11-19T17:50:04,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/00b46f0ce3344805820b4d4e9967c15c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/00b46f0ce3344805820b4d4e9967c15c 2024-11-19T17:50:04,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/24ec8f2c957b4b5ca2c7f77aaaa732d8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/24ec8f2c957b4b5ca2c7f77aaaa732d8 2024-11-19T17:50:04,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ed6561b74a0c4be1ad628835f0898c3c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ed6561b74a0c4be1ad628835f0898c3c 2024-11-19T17:50:04,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4c3cc264580c40ae9b2958f53f1274ef to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4c3cc264580c40ae9b2958f53f1274ef 2024-11-19T17:50:04,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b5017abf812845198674256b41196571 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b5017abf812845198674256b41196571 2024-11-19T17:50:04,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ec69bb1f5c6847569d14fbdbb2c079ec to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/ec69bb1f5c6847569d14fbdbb2c079ec 2024-11-19T17:50:04,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8619ddca353f4031a1174944d75d2490 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8619ddca353f4031a1174944d75d2490 2024-11-19T17:50:04,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8b39c8c71578443f89547eb2451bb8e7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/8b39c8c71578443f89547eb2451bb8e7 2024-11-19T17:50:04,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d9b5e93d154c8baf8361512ab73503 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/c2d9b5e93d154c8baf8361512ab73503 2024-11-19T17:50:04,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/0526182ea9f84301aa152dd59956b820 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/0526182ea9f84301aa152dd59956b820 2024-11-19T17:50:04,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cfb70028a7c64653ad3b534cb170e7a6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cfb70028a7c64653ad3b534cb170e7a6 2024-11-19T17:50:04,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/b1e7e41d52a347ebb0dce2b392a13152, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d8c311d6a8d442348dd778e9848b5925, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ffdd03fa408e446f827b83fa299ef682, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/3c06259f238d44a8a88d6e4c77dfe30b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/03576248eb30461f80119009056471c6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/75a903a2f1be4505879c15ceb093098c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9d60fab0f96547d9944cd279a880c27d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/a6262caf707a4cc3a371c47fcf71d077, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d9e7dbc29309474b8dc4f0e448df6078, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/1cc4161b27a64fc48da92533b47769e0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cd7daf9b4bf1473cbbac4c1c92fbd70c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9de6ef3fc15f4415a01f6e7813ce2cd1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/e696306753024f50974065cf46c6cfe2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/fdc4d515ef644dc89544efa2046369cd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ec17ed6f8320430b8d51c0badbff3ac1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/652c0a33f3bb43d28c67f304140b25e0] to archive 2024-11-19T17:50:04,520 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:04,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/b1e7e41d52a347ebb0dce2b392a13152 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/b1e7e41d52a347ebb0dce2b392a13152 2024-11-19T17:50:04,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d8c311d6a8d442348dd778e9848b5925 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d8c311d6a8d442348dd778e9848b5925 2024-11-19T17:50:04,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ffdd03fa408e446f827b83fa299ef682 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ffdd03fa408e446f827b83fa299ef682 2024-11-19T17:50:04,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/3c06259f238d44a8a88d6e4c77dfe30b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/3c06259f238d44a8a88d6e4c77dfe30b 2024-11-19T17:50:04,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/03576248eb30461f80119009056471c6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/03576248eb30461f80119009056471c6 2024-11-19T17:50:04,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/75a903a2f1be4505879c15ceb093098c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/75a903a2f1be4505879c15ceb093098c 2024-11-19T17:50:04,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9d60fab0f96547d9944cd279a880c27d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9d60fab0f96547d9944cd279a880c27d 2024-11-19T17:50:04,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/a6262caf707a4cc3a371c47fcf71d077 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/a6262caf707a4cc3a371c47fcf71d077 2024-11-19T17:50:04,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d9e7dbc29309474b8dc4f0e448df6078 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/d9e7dbc29309474b8dc4f0e448df6078 2024-11-19T17:50:04,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/1cc4161b27a64fc48da92533b47769e0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/1cc4161b27a64fc48da92533b47769e0 2024-11-19T17:50:04,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cd7daf9b4bf1473cbbac4c1c92fbd70c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cd7daf9b4bf1473cbbac4c1c92fbd70c 2024-11-19T17:50:04,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9de6ef3fc15f4415a01f6e7813ce2cd1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9de6ef3fc15f4415a01f6e7813ce2cd1 2024-11-19T17:50:04,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/e696306753024f50974065cf46c6cfe2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/e696306753024f50974065cf46c6cfe2 2024-11-19T17:50:04,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/fdc4d515ef644dc89544efa2046369cd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/fdc4d515ef644dc89544efa2046369cd 2024-11-19T17:50:04,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ec17ed6f8320430b8d51c0badbff3ac1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/ec17ed6f8320430b8d51c0badbff3ac1 2024-11-19T17:50:04,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/652c0a33f3bb43d28c67f304140b25e0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/652c0a33f3bb43d28c67f304140b25e0 2024-11-19T17:50:04,689 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/b9ca6e7072c14ed49a65358a61e09be4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b9ca6e7072c14ed49a65358a61e09be4 2024-11-19T17:50:04,693 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/B of 47a3c46e7ce46ae36be91291b307632c into b9ca6e7072c14ed49a65358a61e09be4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:04,693 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:04,693 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/B, priority=12, startTime=1732038604272; duration=0sec 2024-11-19T17:50:04,693 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:04,693 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:B 2024-11-19T17:50:04,694 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:04,695 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:04,695 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 47a3c46e7ce46ae36be91291b307632c/C is initiating minor compaction (all files) 2024-11-19T17:50:04,695 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47a3c46e7ce46ae36be91291b307632c/C in TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:04,695 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9bb7ce1b193d4ce8905c14228b99b8e8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/67c57439697047db9cf9f942fe596a7f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/88470a09b98547e48c8d242aec97e417, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/08d8bb6816a84146a8b945031b00931b] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp, totalSize=48.3 K 2024-11-19T17:50:04,695 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bb7ce1b193d4ce8905c14228b99b8e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732038598784 2024-11-19T17:50:04,695 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67c57439697047db9cf9f942fe596a7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732038599158 2024-11-19T17:50:04,696 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88470a09b98547e48c8d242aec97e417, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732038600728 2024-11-19T17:50:04,696 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08d8bb6816a84146a8b945031b00931b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038601919 2024-11-19T17:50:04,704 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#C#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:04,705 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/0fdfe9870b784b10b0b65c7a6e2e2482 is 50, key is test_row_0/C:col10/1732038601919/Put/seqid=0 2024-11-19T17:50:04,711 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47a3c46e7ce46ae36be91291b307632c#A#compaction#373 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:04,711 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/ad40bbf32a2440ae869bd281c9da0507 is 175, key is test_row_0/A:col10/1732038601919/Put/seqid=0 2024-11-19T17:50:04,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742270_1446 (size=12439) 2024-11-19T17:50:04,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742271_1447 (size=31393) 2024-11-19T17:50:04,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:04,802 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194922130484034599b8494ddbf208c656_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194922130484034599b8494ddbf208c656_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/f6f08c6c38db4b248e3d6974d9022a81, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/f6f08c6c38db4b248e3d6974d9022a81 is 175, key is test_row_0/A:col10/1732038602648/Put/seqid=0 2024-11-19T17:50:04,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742272_1448 (size=31255) 2024-11-19T17:50:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:04,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. as already flushing 2024-11-19T17:50:04,830 INFO [Thread-1639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1639. 2024-11-19T17:50:04,830 INFO [Thread-1645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1645. 2024-11-19T17:50:04,831 DEBUG [Thread-1639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:04,831 DEBUG [Thread-1645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:04,831 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:04,831 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:04,831 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:04,831 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:04,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:04,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:50:05,000 INFO [Thread-1641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1641. 2024-11-19T17:50:05,000 DEBUG [Thread-1641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:05,000 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:05,000 INFO [Thread-1643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1643. 2024-11-19T17:50:05,000 DEBUG [Thread-1643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:05,000 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:05,000 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:05,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:05,000 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:05,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:05,126 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/0fdfe9870b784b10b0b65c7a6e2e2482 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/0fdfe9870b784b10b0b65c7a6e2e2482 2024-11-19T17:50:05,132 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/C of 47a3c46e7ce46ae36be91291b307632c into 0fdfe9870b784b10b0b65c7a6e2e2482(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:05,132 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:05,132 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/C, priority=12, startTime=1732038604272; duration=0sec 2024-11-19T17:50:05,132 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:05,132 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:C 2024-11-19T17:50:05,133 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/ad40bbf32a2440ae869bd281c9da0507 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/ad40bbf32a2440ae869bd281c9da0507 2024-11-19T17:50:05,137 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 47a3c46e7ce46ae36be91291b307632c/A of 47a3c46e7ce46ae36be91291b307632c into ad40bbf32a2440ae869bd281c9da0507(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:05,138 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:05,138 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., storeName=47a3c46e7ce46ae36be91291b307632c/A, priority=12, startTime=1732038604272; duration=0sec 2024-11-19T17:50:05,138 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:05,138 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47a3c46e7ce46ae36be91291b307632c:A 2024-11-19T17:50:05,208 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/f6f08c6c38db4b248e3d6974d9022a81 2024-11-19T17:50:05,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/4ea144a8f8884640bc24a68f2b08c0ed is 50, key is test_row_0/B:col10/1732038602648/Put/seqid=0 2024-11-19T17:50:05,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742273_1449 (size=12301) 2024-11-19T17:50:05,218 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/4ea144a8f8884640bc24a68f2b08c0ed 2024-11-19T17:50:05,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/7cf28f88925f483faa5881f483742005 is 50, key is test_row_0/C:col10/1732038602648/Put/seqid=0 2024-11-19T17:50:05,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742274_1450 (size=12301) 2024-11-19T17:50:05,628 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/7cf28f88925f483faa5881f483742005 2024-11-19T17:50:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/f6f08c6c38db4b248e3d6974d9022a81 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/f6f08c6c38db4b248e3d6974d9022a81 2024-11-19T17:50:05,636 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/f6f08c6c38db4b248e3d6974d9022a81, entries=150, sequenceid=311, filesize=30.5 K 2024-11-19T17:50:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/4ea144a8f8884640bc24a68f2b08c0ed as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4ea144a8f8884640bc24a68f2b08c0ed 2024-11-19T17:50:05,642 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4ea144a8f8884640bc24a68f2b08c0ed, entries=150, sequenceid=311, filesize=12.0 K 2024-11-19T17:50:05,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/7cf28f88925f483faa5881f483742005 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/7cf28f88925f483faa5881f483742005 2024-11-19T17:50:05,647 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/7cf28f88925f483faa5881f483742005, entries=150, sequenceid=311, filesize=12.0 K 2024-11-19T17:50:05,648 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=26.84 KB/27480 for 47a3c46e7ce46ae36be91291b307632c in 1261ms, sequenceid=311, compaction requested=false 2024-11-19T17:50:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 47a3c46e7ce46ae36be91291b307632c: 2024-11-19T17:50:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-19T17:50:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-19T17:50:05,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T17:50:05,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7960 sec 2024-11-19T17:50:05,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.8040 sec 2024-11-19T17:50:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:50:07,019 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:07,441 INFO [Thread-1647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1647. 2024-11-19T17:50:07,441 DEBUG [Thread-1647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:07,441 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:07,441 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:07,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1381 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4141 rows 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1380 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4134 rows 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1387 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4159 rows 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1379 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4137 rows 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1383 2024-11-19T17:50:07,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4143 rows 2024-11-19T17:50:07,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:50:07,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:50:07,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:07,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:07,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:07,443 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:50:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:50:07,450 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038607450"}]},"ts":"1732038607450"} 2024-11-19T17:50:07,452 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:50:07,452 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:50:07,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:50:07,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, UNASSIGN}] 2024-11-19T17:50:07,455 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, UNASSIGN 2024-11-19T17:50:07,458 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=47a3c46e7ce46ae36be91291b307632c, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:07,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, UNASSIGN because future has completed 2024-11-19T17:50:07,460 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:50:07,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:50:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:50:07,612 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:07,612 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:50:07,612 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing 47a3c46e7ce46ae36be91291b307632c, disabling compactions & flushes 2024-11-19T17:50:07,612 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:07,612 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:07,612 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. after waiting 0 ms 2024-11-19T17:50:07,612 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:07,612 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing 47a3c46e7ce46ae36be91291b307632c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T17:50:07,618 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190d998145c61a404697b4763c31505a9e_47a3c46e7ce46ae36be91291b307632c is 50, key is test_row_0/A:col10/1732038607439/Put/seqid=0 2024-11-19T17:50:07,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742275_1451 (size=12454) 2024-11-19T17:50:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:50:08,021 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:08,025 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190d998145c61a404697b4763c31505a9e_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d998145c61a404697b4763c31505a9e_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:08,025 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/e011ad9bb4344ff1b8f70175b577a33d, store: [table=TestAcidGuarantees family=A region=47a3c46e7ce46ae36be91291b307632c] 2024-11-19T17:50:08,026 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/e011ad9bb4344ff1b8f70175b577a33d is 175, key is test_row_0/A:col10/1732038607439/Put/seqid=0 2024-11-19T17:50:08,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742276_1452 (size=31255) 2024-11-19T17:50:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:50:08,431 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/e011ad9bb4344ff1b8f70175b577a33d 2024-11-19T17:50:08,444 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/cd8a8bfd1d8d49e397eccec4e8dea4aa is 50, key is test_row_0/B:col10/1732038607439/Put/seqid=0 2024-11-19T17:50:08,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742277_1453 (size=12301) 2024-11-19T17:50:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:50:08,854 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/cd8a8bfd1d8d49e397eccec4e8dea4aa 2024-11-19T17:50:08,860 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/cab1a6c438dc4aa39510e64d35656fe0 is 50, key is test_row_0/C:col10/1732038607439/Put/seqid=0 2024-11-19T17:50:08,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742278_1454 (size=12301) 2024-11-19T17:50:09,263 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/cab1a6c438dc4aa39510e64d35656fe0 2024-11-19T17:50:09,267 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/A/e011ad9bb4344ff1b8f70175b577a33d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/e011ad9bb4344ff1b8f70175b577a33d 2024-11-19T17:50:09,269 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/e011ad9bb4344ff1b8f70175b577a33d, entries=150, sequenceid=322, filesize=30.5 K 2024-11-19T17:50:09,270 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/B/cd8a8bfd1d8d49e397eccec4e8dea4aa as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cd8a8bfd1d8d49e397eccec4e8dea4aa 2024-11-19T17:50:09,272 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cd8a8bfd1d8d49e397eccec4e8dea4aa, entries=150, sequenceid=322, filesize=12.0 K 2024-11-19T17:50:09,273 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/.tmp/C/cab1a6c438dc4aa39510e64d35656fe0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cab1a6c438dc4aa39510e64d35656fe0 2024-11-19T17:50:09,275 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cab1a6c438dc4aa39510e64d35656fe0, entries=150, sequenceid=322, filesize=12.0 K 2024-11-19T17:50:09,276 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 47a3c46e7ce46ae36be91291b307632c in 1664ms, sequenceid=322, compaction requested=true 2024-11-19T17:50:09,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fc47d9d6030d496cae900edfb6d3c670, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/598cc11c03614ab4b2e7ab04193d01a7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/53bbeacddae34c7aa02146e6698263a3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/bfe2ccbc36e64a288ad1a027d92f4eef] to archive 2024-11-19T17:50:09,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:09,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fc47d9d6030d496cae900edfb6d3c670 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/fc47d9d6030d496cae900edfb6d3c670 2024-11-19T17:50:09,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/598cc11c03614ab4b2e7ab04193d01a7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/598cc11c03614ab4b2e7ab04193d01a7 2024-11-19T17:50:09,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/53bbeacddae34c7aa02146e6698263a3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/53bbeacddae34c7aa02146e6698263a3 2024-11-19T17:50:09,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/bfe2ccbc36e64a288ad1a027d92f4eef to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/bfe2ccbc36e64a288ad1a027d92f4eef 2024-11-19T17:50:09,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/080accc7b72b457b8e54f3cadbd2923c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/bfbe069ace284b69939d0e70d84fbbdc, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/d86670910ef24dd1b203de648854163b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7a04dc215dae45edb32ba8d6f0166986] to archive 2024-11-19T17:50:09,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:09,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/080accc7b72b457b8e54f3cadbd2923c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/080accc7b72b457b8e54f3cadbd2923c 2024-11-19T17:50:09,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/bfbe069ace284b69939d0e70d84fbbdc to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/bfbe069ace284b69939d0e70d84fbbdc 2024-11-19T17:50:09,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/d86670910ef24dd1b203de648854163b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/d86670910ef24dd1b203de648854163b 2024-11-19T17:50:09,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7a04dc215dae45edb32ba8d6f0166986 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/7a04dc215dae45edb32ba8d6f0166986 2024-11-19T17:50:09,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9bb7ce1b193d4ce8905c14228b99b8e8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/67c57439697047db9cf9f942fe596a7f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/88470a09b98547e48c8d242aec97e417, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/08d8bb6816a84146a8b945031b00931b] to archive 2024-11-19T17:50:09,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:09,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9bb7ce1b193d4ce8905c14228b99b8e8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/9bb7ce1b193d4ce8905c14228b99b8e8 2024-11-19T17:50:09,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/67c57439697047db9cf9f942fe596a7f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/67c57439697047db9cf9f942fe596a7f 2024-11-19T17:50:09,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/88470a09b98547e48c8d242aec97e417 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/88470a09b98547e48c8d242aec97e417 2024-11-19T17:50:09,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/08d8bb6816a84146a8b945031b00931b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/08d8bb6816a84146a8b945031b00931b 2024-11-19T17:50:09,291 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/recovered.edits/325.seqid, newMaxSeqId=325, maxSeqId=4 2024-11-19T17:50:09,291 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. 2024-11-19T17:50:09,291 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for 47a3c46e7ce46ae36be91291b307632c: Waiting for close lock at 1732038607612Running coprocessor pre-close hooks at 1732038607612Disabling compacts and flushes for region at 1732038607612Disabling writes for close at 1732038607612Obtaining lock to block concurrent updates at 1732038607612Preparing flush snapshotting stores in 47a3c46e7ce46ae36be91291b307632c at 1732038607612Finished memstore snapshotting TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732038607613 (+1 ms)Flushing stores of TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c. at 1732038607613Flushing 47a3c46e7ce46ae36be91291b307632c/A: creating writer at 1732038607613Flushing mob file 47a3c46e7ce46ae36be91291b307632c/A: appending metadata at 1732038607618 (+5 ms)Flushing mob file 47a3c46e7ce46ae36be91291b307632c/A: closing flushed file at 1732038607618Flushing 47a3c46e7ce46ae36be91291b307632c/A: appending metadata at 1732038608026 (+408 ms)Flushing 47a3c46e7ce46ae36be91291b307632c/A: closing flushed file at 1732038608026Flushing 47a3c46e7ce46ae36be91291b307632c/B: creating writer at 1732038608435 (+409 ms)Flushing 47a3c46e7ce46ae36be91291b307632c/B: appending metadata at 1732038608444 (+9 ms)Flushing 47a3c46e7ce46ae36be91291b307632c/B: closing flushed file at 1732038608444Flushing 47a3c46e7ce46ae36be91291b307632c/C: creating writer at 1732038608857 (+413 ms)Flushing 47a3c46e7ce46ae36be91291b307632c/C: appending metadata at 1732038608859 (+2 ms)Flushing 47a3c46e7ce46ae36be91291b307632c/C: closing flushed file at 1732038608859Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eec5ee8: reopening flushed file at 1732038609266 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@219287c7: reopening flushed file at 1732038609269 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fba5b48: reopening flushed file at 1732038609272 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 47a3c46e7ce46ae36be91291b307632c in 1664ms, sequenceid=322, compaction requested=true at 1732038609276 (+4 ms)Writing region close event to WAL at 1732038609289 (+13 ms)Running coprocessor post-close hooks at 1732038609291 (+2 ms)Closed at 1732038609291 2024-11-19T17:50:09,293 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed 47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,293 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=47a3c46e7ce46ae36be91291b307632c, regionState=CLOSED 2024-11-19T17:50:09,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:50:09,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T17:50:09,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure 47a3c46e7ce46ae36be91291b307632c, server=26d3106abcfe,36839,1732038481522 in 1.8350 sec 2024-11-19T17:50:09,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-19T17:50:09,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47a3c46e7ce46ae36be91291b307632c, UNASSIGN in 1.8420 sec 2024-11-19T17:50:09,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T17:50:09,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8450 sec 2024-11-19T17:50:09,301 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038609301"}]},"ts":"1732038609301"} 2024-11-19T17:50:09,302 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:50:09,302 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:50:09,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8590 sec 2024-11-19T17:50:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:50:09,600 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:09,601 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:50:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:09,604 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:50:09,605 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:09,607 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,608 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/recovered.edits] 2024-11-19T17:50:09,610 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/ad40bbf32a2440ae869bd281c9da0507 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/ad40bbf32a2440ae869bd281c9da0507 2024-11-19T17:50:09,611 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/e011ad9bb4344ff1b8f70175b577a33d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/e011ad9bb4344ff1b8f70175b577a33d 2024-11-19T17:50:09,612 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/f6f08c6c38db4b248e3d6974d9022a81 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/A/f6f08c6c38db4b248e3d6974d9022a81 2024-11-19T17:50:09,615 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4ea144a8f8884640bc24a68f2b08c0ed to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/4ea144a8f8884640bc24a68f2b08c0ed 2024-11-19T17:50:09,616 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b9ca6e7072c14ed49a65358a61e09be4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/b9ca6e7072c14ed49a65358a61e09be4 2024-11-19T17:50:09,617 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cd8a8bfd1d8d49e397eccec4e8dea4aa to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/B/cd8a8bfd1d8d49e397eccec4e8dea4aa 2024-11-19T17:50:09,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/0fdfe9870b784b10b0b65c7a6e2e2482 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/0fdfe9870b784b10b0b65c7a6e2e2482 2024-11-19T17:50:09,620 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/7cf28f88925f483faa5881f483742005 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/7cf28f88925f483faa5881f483742005 2024-11-19T17:50:09,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cab1a6c438dc4aa39510e64d35656fe0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/C/cab1a6c438dc4aa39510e64d35656fe0 2024-11-19T17:50:09,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/recovered.edits/325.seqid to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c/recovered.edits/325.seqid 2024-11-19T17:50:09,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,623 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:50:09,623 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:50:09,624 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T17:50:09,626 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190c3e04e8a4154ce18d906a1af82410ea_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190c3e04e8a4154ce18d906a1af82410ea_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,627 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d998145c61a404697b4763c31505a9e_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d998145c61a404697b4763c31505a9e_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,628 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c5393ee6bf844428b5d82f382740d86_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c5393ee6bf844428b5d82f382740d86_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,628 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119306380d9819d4b018413ccc564776737_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119306380d9819d4b018413ccc564776737_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,629 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939f9c2b9723b4cfeb586e906cff9d99a_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939f9c2b9723b4cfeb586e906cff9d99a_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,630 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193fe33155743b4e32860d271ec9417be2_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193fe33155743b4e32860d271ec9417be2_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,630 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194922130484034599b8494ddbf208c656_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194922130484034599b8494ddbf208c656_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,631 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be91cf408c1471db8d9ee5484ee2fed_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be91cf408c1471db8d9ee5484ee2fed_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,632 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965961b3478cc423380988001ee7b94c0_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965961b3478cc423380988001ee7b94c0_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,632 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198338aa1dc25b40cd94984c1cabdff4e6_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198338aa1dc25b40cd94984c1cabdff4e6_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,633 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198634ba8b3e184cf7bb17107290836fad_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198634ba8b3e184cf7bb17107290836fad_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,633 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a563a89783444714ba2c0e63794703fc_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a563a89783444714ba2c0e63794703fc_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,634 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a602aafbf1b344db890a3790910b5073_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a602aafbf1b344db890a3790910b5073_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,635 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c477037a15d041a8b171c68b66f88cba_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c477037a15d041a8b171c68b66f88cba_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,635 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e799b468ae6b4a0b85706ea3fd5cf873_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e799b468ae6b4a0b85706ea3fd5cf873_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,636 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e87dc538af8742d9a4672134c4aa8c28_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e87dc538af8742d9a4672134c4aa8c28_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,637 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f1d9b0f45a434db29d6db67bc2060e5d_47a3c46e7ce46ae36be91291b307632c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f1d9b0f45a434db29d6db67bc2060e5d_47a3c46e7ce46ae36be91291b307632c 2024-11-19T17:50:09,637 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:50:09,639 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:09,640 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:50:09,642 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:50:09,643 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:09,643 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:50:09,643 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732038609643"}]},"ts":"9223372036854775807"} 2024-11-19T17:50:09,645 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:50:09,645 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 47a3c46e7ce46ae36be91291b307632c, NAME => 'TestAcidGuarantees,,1732038580574.47a3c46e7ce46ae36be91291b307632c.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:50:09,645 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:50:09,645 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732038609645"}]},"ts":"9223372036854775807"} 2024-11-19T17:50:09,646 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:50:09,647 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:09,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-19T17:50:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:50:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:50:09,711 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:09,717 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-548734850_22 at /127.0.0.1:49768 [Waiting for operation #332] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1036582635_22 at /127.0.0.1:35266 [Waiting for operation #77] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=644 (was 551) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9853 (was 9498) - AvailableMemoryMB LEAK? - 2024-11-19T17:50:09,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=644, ProcessCount=11, AvailableMemoryMB=9867 2024-11-19T17:50:09,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:50:09,726 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:50:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:09,728 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:50:09,728 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:09,728 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-19T17:50:09,728 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:50:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-19T17:50:09,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742279_1455 (size=959) 2024-11-19T17:50:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-19T17:50:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-19T17:50:10,134 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d 2024-11-19T17:50:10,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742280_1456 (size=53) 2024-11-19T17:50:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-19T17:50:10,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:50:10,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 65f2046e0a9c76be2cfb08c4db762a3f, disabling compactions & flushes 2024-11-19T17:50:10,539 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:10,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:10,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. after waiting 0 ms 2024-11-19T17:50:10,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:10,539 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:10,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 65f2046e0a9c76be2cfb08c4db762a3f: Waiting for close lock at 1732038610539Disabling compacts and flushes for region at 1732038610539Disabling writes for close at 1732038610539Writing region close event to WAL at 1732038610539Closed at 1732038610539 2024-11-19T17:50:10,540 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:50:10,541 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732038610540"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732038610540"}]},"ts":"1732038610540"} 2024-11-19T17:50:10,542 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:50:10,543 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:50:10,543 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038610543"}]},"ts":"1732038610543"} 2024-11-19T17:50:10,545 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:50:10,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, ASSIGN}] 2024-11-19T17:50:10,547 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, ASSIGN 2024-11-19T17:50:10,547 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, ASSIGN; state=OFFLINE, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=false 2024-11-19T17:50:10,698 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=65f2046e0a9c76be2cfb08c4db762a3f, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:10,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, ASSIGN because future has completed 2024-11-19T17:50:10,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:50:10,854 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:10,854 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:50:10,855 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,855 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:50:10,855 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,855 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,856 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,857 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f2046e0a9c76be2cfb08c4db762a3f columnFamilyName A 2024-11-19T17:50:10,857 DEBUG [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:10,858 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] regionserver.HStore(327): Store=65f2046e0a9c76be2cfb08c4db762a3f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:10,858 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,859 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f2046e0a9c76be2cfb08c4db762a3f columnFamilyName B 2024-11-19T17:50:10,859 DEBUG [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:10,859 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] regionserver.HStore(327): Store=65f2046e0a9c76be2cfb08c4db762a3f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:10,859 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,860 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f2046e0a9c76be2cfb08c4db762a3f columnFamilyName C 2024-11-19T17:50:10,860 DEBUG [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:10,861 INFO [StoreOpener-65f2046e0a9c76be2cfb08c4db762a3f-1 {}] regionserver.HStore(327): Store=65f2046e0a9c76be2cfb08c4db762a3f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:10,861 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,861 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,862 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,862 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,862 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,863 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:50:10,864 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,865 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:50:10,866 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 65f2046e0a9c76be2cfb08c4db762a3f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61219098, jitterRate=-0.08776435256004333}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:50:10,866 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:10,866 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 65f2046e0a9c76be2cfb08c4db762a3f: Running coprocessor pre-open hook at 1732038610855Writing region info on filesystem at 1732038610855Initializing all the Stores at 1732038610856 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038610856Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038610856Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038610856Cleaning up temporary data from old regions at 1732038610862 (+6 ms)Running coprocessor post-open hooks at 1732038610866 (+4 ms)Region opened successfully at 1732038610866 2024-11-19T17:50:10,867 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., pid=124, masterSystemTime=1732038610852 2024-11-19T17:50:10,868 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:10,868 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-19T17:50:10,869 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=65f2046e0a9c76be2cfb08c4db762a3f, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:10,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:50:10,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T17:50:10,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 in 170 msec 2024-11-19T17:50:10,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-19T17:50:10,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, ASSIGN in 327 msec 2024-11-19T17:50:10,874 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:50:10,874 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038610874"}]},"ts":"1732038610874"} 2024-11-19T17:50:10,875 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:50:10,875 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:50:10,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-19T17:50:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-19T17:50:11,879 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:11,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,883 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18af1d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,885 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33080c14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,887 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,890 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:50:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,41481,1732038480627 2024-11-19T17:50:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b88d04e 2024-11-19T17:50:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:50:11,892 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:50:11,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,898 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d26ea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,900 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d247462, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,901 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e507724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec78b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,904 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d2a60fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,905 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bedfd69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,907 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d453d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,908 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d63697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,909 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bae0706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,911 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d76a301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,912 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769a8967, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,914 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5221a249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,916 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7984926a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,918 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a942d9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,920 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179972dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,921 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bcdcbf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,923 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f290d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@188457d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:11,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:11,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bc647f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:11,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e1c7183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:11,928 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,928 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,928 DEBUG [Thread-2046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,928 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,929 DEBUG [Thread-2050 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,929 DEBUG [Thread-2050 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,929 DEBUG [Thread-2044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,929 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,930 DEBUG [Thread-2042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,932 DEBUG [Thread-2048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T17:50:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,934 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T17:50:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,935 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:11,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:11,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,938 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:11,940 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:11,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2] 2024-11-19T17:50:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:11,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:50:11,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/950afa748eb748d8a929e5bf0d8cec1b is 50, key is test_row_0/A:col10/1732038611943/Put/seqid=0 2024-11-19T17:50:12,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038671997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742281_1457 (size=12001) 2024-11-19T17:50:12,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038672000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/950afa748eb748d8a929e5bf0d8cec1b 2024-11-19T17:50:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038672005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038672008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038672008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/08b68c2e74a64d57b7939dc477aa20f1 is 50, key is test_row_0/B:col10/1732038611943/Put/seqid=0 2024-11-19T17:50:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T17:50:12,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742282_1458 (size=12001) 2024-11-19T17:50:12,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/08b68c2e74a64d57b7939dc477aa20f1 2024-11-19T17:50:12,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T17:50:12,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:12,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:12,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:12,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:12,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:12,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/55acbdab0134482a929ac8c9ae0da508 is 50, key is test_row_0/C:col10/1732038611943/Put/seqid=0 2024-11-19T17:50:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038672119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038672119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038672119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038672120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038672128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742283_1459 (size=12001) 2024-11-19T17:50:12,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/55acbdab0134482a929ac8c9ae0da508 2024-11-19T17:50:12,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/950afa748eb748d8a929e5bf0d8cec1b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/950afa748eb748d8a929e5bf0d8cec1b 2024-11-19T17:50:12,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/950afa748eb748d8a929e5bf0d8cec1b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T17:50:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/08b68c2e74a64d57b7939dc477aa20f1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/08b68c2e74a64d57b7939dc477aa20f1 2024-11-19T17:50:12,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/08b68c2e74a64d57b7939dc477aa20f1, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T17:50:12,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/55acbdab0134482a929ac8c9ae0da508 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/55acbdab0134482a929ac8c9ae0da508 2024-11-19T17:50:12,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/55acbdab0134482a929ac8c9ae0da508, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T17:50:12,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 65f2046e0a9c76be2cfb08c4db762a3f in 212ms, sequenceid=12, compaction requested=false 2024-11-19T17:50:12,159 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:50:12,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:12,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T17:50:12,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:12,254 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T17:50:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T17:50:12,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ef6d51b75ffb4c808a173380f73f179e is 50, key is test_row_0/A:col10/1732038612005/Put/seqid=0 2024-11-19T17:50:12,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742284_1460 (size=12001) 2024-11-19T17:50:12,291 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ef6d51b75ffb4c808a173380f73f179e 2024-11-19T17:50:12,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c0501ee838ab4027a099000a3d298993 is 50, key is test_row_0/B:col10/1732038612005/Put/seqid=0 2024-11-19T17:50:12,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742285_1461 (size=12001) 2024-11-19T17:50:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:12,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038672345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038672347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038672347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038672349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038672349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038672459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038672469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038672469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038672469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038672469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T17:50:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038672679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038672679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038672689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038672689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038672690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,706 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c0501ee838ab4027a099000a3d298993 2024-11-19T17:50:12,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/597059c295664237980baf5899d89c7f is 50, key is test_row_0/C:col10/1732038612005/Put/seqid=0 2024-11-19T17:50:12,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742286_1462 (size=12001) 2024-11-19T17:50:12,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038672988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:12,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038672988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:12,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038672999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038673009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038673009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T17:50:13,116 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/597059c295664237980baf5899d89c7f 2024-11-19T17:50:13,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ef6d51b75ffb4c808a173380f73f179e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef6d51b75ffb4c808a173380f73f179e 2024-11-19T17:50:13,126 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef6d51b75ffb4c808a173380f73f179e, entries=150, sequenceid=38, filesize=11.7 K 2024-11-19T17:50:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c0501ee838ab4027a099000a3d298993 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c0501ee838ab4027a099000a3d298993 2024-11-19T17:50:13,131 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c0501ee838ab4027a099000a3d298993, entries=150, sequenceid=38, filesize=11.7 K 2024-11-19T17:50:13,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/597059c295664237980baf5899d89c7f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/597059c295664237980baf5899d89c7f 2024-11-19T17:50:13,135 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/597059c295664237980baf5899d89c7f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-19T17:50:13,136 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 65f2046e0a9c76be2cfb08c4db762a3f in 883ms, sequenceid=38, compaction requested=false 2024-11-19T17:50:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:13,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:13,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T17:50:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T17:50:13,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T17:50:13,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1990 sec 2024-11-19T17:50:13,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.2090 sec 2024-11-19T17:50:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:13,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:50:13,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ae12e7bb363543ef899f925a589509b7 is 50, key is test_row_0/A:col10/1732038612347/Put/seqid=0 2024-11-19T17:50:13,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742287_1463 (size=14341) 2024-11-19T17:50:13,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ae12e7bb363543ef899f925a589509b7 2024-11-19T17:50:13,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/0c9b946ad40f4b268a5e488b6b383b3f is 50, key is test_row_0/B:col10/1732038612347/Put/seqid=0 2024-11-19T17:50:13,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742288_1464 (size=12001) 2024-11-19T17:50:13,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/0c9b946ad40f4b268a5e488b6b383b3f 2024-11-19T17:50:13,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/27ba2a4f9f154e3b88dc749badb6884c is 50, key is test_row_0/C:col10/1732038612347/Put/seqid=0 2024-11-19T17:50:13,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742289_1465 (size=12001) 2024-11-19T17:50:13,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/27ba2a4f9f154e3b88dc749badb6884c 2024-11-19T17:50:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ae12e7bb363543ef899f925a589509b7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ae12e7bb363543ef899f925a589509b7 2024-11-19T17:50:13,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ae12e7bb363543ef899f925a589509b7, entries=200, sequenceid=50, filesize=14.0 K 2024-11-19T17:50:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/0c9b946ad40f4b268a5e488b6b383b3f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0c9b946ad40f4b268a5e488b6b383b3f 2024-11-19T17:50:13,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0c9b946ad40f4b268a5e488b6b383b3f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T17:50:13,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/27ba2a4f9f154e3b88dc749badb6884c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/27ba2a4f9f154e3b88dc749badb6884c 2024-11-19T17:50:13,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/27ba2a4f9f154e3b88dc749badb6884c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T17:50:13,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=114.05 KB/116790 for 65f2046e0a9c76be2cfb08c4db762a3f in 87ms, sequenceid=50, compaction requested=true 2024-11-19T17:50:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:13,602 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:13,602 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:13,604 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:13,604 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/A is initiating minor compaction (all files) 2024-11-19T17:50:13,605 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/A in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:13,605 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/950afa748eb748d8a929e5bf0d8cec1b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef6d51b75ffb4c808a173380f73f179e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ae12e7bb363543ef899f925a589509b7] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=37.4 K 2024-11-19T17:50:13,605 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:13,605 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/B is initiating minor compaction (all files) 2024-11-19T17:50:13,605 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/B in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:13,605 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/08b68c2e74a64d57b7939dc477aa20f1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c0501ee838ab4027a099000a3d298993, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0c9b946ad40f4b268a5e488b6b383b3f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=35.2 K 2024-11-19T17:50:13,605 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 950afa748eb748d8a929e5bf0d8cec1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732038611939 2024-11-19T17:50:13,605 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b68c2e74a64d57b7939dc477aa20f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732038611939 2024-11-19T17:50:13,606 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6d51b75ffb4c808a173380f73f179e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732038612004 2024-11-19T17:50:13,606 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c0501ee838ab4027a099000a3d298993, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732038612004 2024-11-19T17:50:13,606 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae12e7bb363543ef899f925a589509b7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732038612333 2024-11-19T17:50:13,606 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9b946ad40f4b268a5e488b6b383b3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732038612333 2024-11-19T17:50:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:13,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T17:50:13,614 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:13,615 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/4d7a26744a3e423fabae9b722bea6151 is 50, key is test_row_0/A:col10/1732038612347/Put/seqid=0 2024-11-19T17:50:13,617 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#B#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:13,618 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/baa5ab55db46463e956212f8988e9eef is 50, key is test_row_0/B:col10/1732038612347/Put/seqid=0 2024-11-19T17:50:13,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/dc6e8689e67e48a09c2788790671f803 is 50, key is test_row_0/A:col10/1732038613601/Put/seqid=0 2024-11-19T17:50:13,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038673626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038673627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038673628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742291_1467 (size=12104) 2024-11-19T17:50:13,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038673629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742290_1466 (size=12104) 2024-11-19T17:50:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038673633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742292_1468 (size=12001) 2024-11-19T17:50:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,647 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/baa5ab55db46463e956212f8988e9eef as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/baa5ab55db46463e956212f8988e9eef 2024-11-19T17:50:13,653 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/B of 65f2046e0a9c76be2cfb08c4db762a3f into baa5ab55db46463e956212f8988e9eef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:13,653 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:13,653 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/B, priority=13, startTime=1732038613602; duration=0sec 2024-11-19T17:50:13,653 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:13,653 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:B 2024-11-19T17:50:13,653 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:13,655 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:13,655 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/C is initiating minor compaction (all files) 2024-11-19T17:50:13,655 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/C in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:13,655 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/55acbdab0134482a929ac8c9ae0da508, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/597059c295664237980baf5899d89c7f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/27ba2a4f9f154e3b88dc749badb6884c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=35.2 K 2024-11-19T17:50:13,655 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 55acbdab0134482a929ac8c9ae0da508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732038611939 2024-11-19T17:50:13,656 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 597059c295664237980baf5899d89c7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732038612004 2024-11-19T17:50:13,656 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 27ba2a4f9f154e3b88dc749badb6884c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732038612333 2024-11-19T17:50:13,662 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#C#compaction#393 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:13,662 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/7535ed8f75cd47e2952707b9ca2e8a38 is 50, key is test_row_0/C:col10/1732038612347/Put/seqid=0 2024-11-19T17:50:13,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742293_1469 (size=12104) 2024-11-19T17:50:13,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038673748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038673748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038673748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038673748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038673748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038673959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038673959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038673959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038673959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:13,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038673969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:13,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:13,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/dc6e8689e67e48a09c2788790671f803 2024-11-19T17:50:14,049 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/4d7a26744a3e423fabae9b722bea6151 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4d7a26744a3e423fabae9b722bea6151 2024-11-19T17:50:14,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3690097a20b4459c9987675e98173695 is 50, key is test_row_0/B:col10/1732038613601/Put/seqid=0 2024-11-19T17:50:14,054 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/A of 65f2046e0a9c76be2cfb08c4db762a3f into 4d7a26744a3e423fabae9b722bea6151(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:14,054 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:14,054 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/A, priority=13, startTime=1732038613602; duration=0sec 2024-11-19T17:50:14,054 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:14,054 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:A 2024-11-19T17:50:14,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742294_1470 (size=12001) 2024-11-19T17:50:14,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3690097a20b4459c9987675e98173695 2024-11-19T17:50:14,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/6c3c2979f0ce41c98157467d926fdae5 is 50, key is test_row_0/C:col10/1732038613601/Put/seqid=0 2024-11-19T17:50:14,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742295_1471 (size=12001) 2024-11-19T17:50:14,076 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/7535ed8f75cd47e2952707b9ca2e8a38 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/7535ed8f75cd47e2952707b9ca2e8a38 2024-11-19T17:50:14,081 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/C of 65f2046e0a9c76be2cfb08c4db762a3f into 7535ed8f75cd47e2952707b9ca2e8a38(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:14,081 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:14,081 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/C, priority=13, startTime=1732038613602; duration=0sec 2024-11-19T17:50:14,082 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:14,082 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:C 2024-11-19T17:50:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T17:50:14,098 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:14,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T17:50:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T17:50:14,102 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:14,103 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:14,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T17:50:14,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T17:50:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:14,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:14,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038674269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038674269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038674269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038674269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038674288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T17:50:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:14,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T17:50:14,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/6c3c2979f0ce41c98157467d926fdae5 2024-11-19T17:50:14,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/dc6e8689e67e48a09c2788790671f803 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/dc6e8689e67e48a09c2788790671f803 2024-11-19T17:50:14,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/dc6e8689e67e48a09c2788790671f803, entries=150, sequenceid=73, filesize=11.7 K 2024-11-19T17:50:14,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3690097a20b4459c9987675e98173695 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3690097a20b4459c9987675e98173695 2024-11-19T17:50:14,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3690097a20b4459c9987675e98173695, entries=150, sequenceid=73, filesize=11.7 K 2024-11-19T17:50:14,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/6c3c2979f0ce41c98157467d926fdae5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/6c3c2979f0ce41c98157467d926fdae5 2024-11-19T17:50:14,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/6c3c2979f0ce41c98157467d926fdae5, entries=150, sequenceid=73, filesize=11.7 K 2024-11-19T17:50:14,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 65f2046e0a9c76be2cfb08c4db762a3f in 888ms, sequenceid=73, compaction requested=false 2024-11-19T17:50:14,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:14,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T17:50:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:14,562 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:50:14,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ab08836f0f9844d0acc853e9c8898aa8 is 50, key is test_row_0/A:col10/1732038613631/Put/seqid=0 2024-11-19T17:50:14,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742296_1472 (size=12001) 2024-11-19T17:50:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T17:50:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:14,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:14,815 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:50:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038674840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038674841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038674841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038674842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038674847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038674959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038674959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038674959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038674959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038674959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:14,970 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ab08836f0f9844d0acc853e9c8898aa8 2024-11-19T17:50:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/d3b2edb25e864720b638d36821b7a1bf is 50, key is test_row_0/B:col10/1732038613631/Put/seqid=0 2024-11-19T17:50:14,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742297_1473 (size=12001) 2024-11-19T17:50:15,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038675169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038675169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038675169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038675169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038675169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T17:50:15,378 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/d3b2edb25e864720b638d36821b7a1bf 2024-11-19T17:50:15,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/d4683d10f9f54eaaa689eba6d4305805 is 50, key is test_row_0/C:col10/1732038613631/Put/seqid=0 2024-11-19T17:50:15,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742298_1474 (size=12001) 2024-11-19T17:50:15,392 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/d4683d10f9f54eaaa689eba6d4305805 2024-11-19T17:50:15,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ab08836f0f9844d0acc853e9c8898aa8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ab08836f0f9844d0acc853e9c8898aa8 2024-11-19T17:50:15,398 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ab08836f0f9844d0acc853e9c8898aa8, entries=150, sequenceid=89, filesize=11.7 K 2024-11-19T17:50:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/d3b2edb25e864720b638d36821b7a1bf as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d3b2edb25e864720b638d36821b7a1bf 2024-11-19T17:50:15,402 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d3b2edb25e864720b638d36821b7a1bf, entries=150, sequenceid=89, filesize=11.7 K 2024-11-19T17:50:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/d4683d10f9f54eaaa689eba6d4305805 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d4683d10f9f54eaaa689eba6d4305805 2024-11-19T17:50:15,405 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d4683d10f9f54eaaa689eba6d4305805, entries=150, sequenceid=89, filesize=11.7 K 2024-11-19T17:50:15,406 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 65f2046e0a9c76be2cfb08c4db762a3f in 844ms, sequenceid=89, compaction requested=true 2024-11-19T17:50:15,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:15,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:15,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T17:50:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T17:50:15,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T17:50:15,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3040 sec 2024-11-19T17:50:15,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.3110 sec 2024-11-19T17:50:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:15,484 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:15,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:50:15,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ea67f771692644e59acbcf85bd0b7ebd is 50, key is test_row_0/A:col10/1732038615484/Put/seqid=0 2024-11-19T17:50:15,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742299_1475 (size=19021) 2024-11-19T17:50:15,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038675490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038675495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038675495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038675496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038675496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038675598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038675608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038675608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038675608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038675619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038675808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038675818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038675818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038675819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038675838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:15,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ea67f771692644e59acbcf85bd0b7ebd 2024-11-19T17:50:15,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3ba8b101d94c40c09375516b716e750c is 50, key is test_row_0/B:col10/1732038615484/Put/seqid=0 2024-11-19T17:50:15,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742300_1476 (size=12001) 2024-11-19T17:50:16,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038676118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038676128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038676128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038676128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038676158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T17:50:16,249 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:16,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T17:50:16,251 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:50:16,251 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:16,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:16,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3ba8b101d94c40c09375516b716e750c 2024-11-19T17:50:16,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/320d857fffcf4b309b58c5931bae45a3 is 50, key is test_row_0/C:col10/1732038615484/Put/seqid=0 2024-11-19T17:50:16,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742301_1477 (size=12001) 2024-11-19T17:50:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:50:16,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:50:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:50:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:50:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038676628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038676638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038676638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038676649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038676668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:16,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:50:16,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:16,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:16,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/320d857fffcf4b309b58c5931bae45a3 2024-11-19T17:50:16,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ea67f771692644e59acbcf85bd0b7ebd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ea67f771692644e59acbcf85bd0b7ebd 2024-11-19T17:50:16,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ea67f771692644e59acbcf85bd0b7ebd, entries=300, sequenceid=115, filesize=18.6 K 2024-11-19T17:50:16,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3ba8b101d94c40c09375516b716e750c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3ba8b101d94c40c09375516b716e750c 2024-11-19T17:50:16,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3ba8b101d94c40c09375516b716e750c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T17:50:16,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/320d857fffcf4b309b58c5931bae45a3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/320d857fffcf4b309b58c5931bae45a3 2024-11-19T17:50:16,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/320d857fffcf4b309b58c5931bae45a3, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T17:50:16,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 65f2046e0a9c76be2cfb08c4db762a3f in 1243ms, sequenceid=115, compaction requested=true 2024-11-19T17:50:16,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:16,727 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:16,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:16,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:16,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:16,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:16,727 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:16,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:16,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:16,728 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:16,728 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/B is initiating minor compaction (all files) 2024-11-19T17:50:16,728 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/B in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,728 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/baa5ab55db46463e956212f8988e9eef, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3690097a20b4459c9987675e98173695, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d3b2edb25e864720b638d36821b7a1bf, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3ba8b101d94c40c09375516b716e750c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=47.0 K 2024-11-19T17:50:16,728 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:16,728 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/A is initiating minor compaction (all files) 2024-11-19T17:50:16,729 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/A in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,729 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting baa5ab55db46463e956212f8988e9eef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732038612333 2024-11-19T17:50:16,729 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4d7a26744a3e423fabae9b722bea6151, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/dc6e8689e67e48a09c2788790671f803, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ab08836f0f9844d0acc853e9c8898aa8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ea67f771692644e59acbcf85bd0b7ebd] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=53.8 K 2024-11-19T17:50:16,729 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3690097a20b4459c9987675e98173695, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732038613539 2024-11-19T17:50:16,729 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7a26744a3e423fabae9b722bea6151, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732038612333 2024-11-19T17:50:16,729 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b2edb25e864720b638d36821b7a1bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732038613624 2024-11-19T17:50:16,729 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc6e8689e67e48a09c2788790671f803, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732038613539 2024-11-19T17:50:16,729 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab08836f0f9844d0acc853e9c8898aa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732038613624 2024-11-19T17:50:16,729 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ba8b101d94c40c09375516b716e750c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038615483 2024-11-19T17:50:16,729 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea67f771692644e59acbcf85bd0b7ebd, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038614839 2024-11-19T17:50:16,737 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:16,737 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/f19191260672431cb05e1116f9cbab07 is 50, key is test_row_0/B:col10/1732038615484/Put/seqid=0 2024-11-19T17:50:16,738 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:16,738 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/38c9797e72e54beea565730e5fe22a7c is 50, key is test_row_0/A:col10/1732038615484/Put/seqid=0 2024-11-19T17:50:16,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742302_1478 (size=12241) 2024-11-19T17:50:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742303_1479 (size=12241) 2024-11-19T17:50:16,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:50:16,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:16,861 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T17:50:16,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/b3fbdc5809dd4b7f82840047e0431d92 is 50, key is test_row_0/A:col10/1732038615485/Put/seqid=0 2024-11-19T17:50:16,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742304_1480 (size=12001) 2024-11-19T17:50:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:50:17,144 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/f19191260672431cb05e1116f9cbab07 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/f19191260672431cb05e1116f9cbab07 2024-11-19T17:50:17,147 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/38c9797e72e54beea565730e5fe22a7c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/38c9797e72e54beea565730e5fe22a7c 2024-11-19T17:50:17,147 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/B of 65f2046e0a9c76be2cfb08c4db762a3f into f19191260672431cb05e1116f9cbab07(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:17,147 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:17,147 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/B, priority=12, startTime=1732038616727; duration=0sec 2024-11-19T17:50:17,148 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:17,148 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:B 2024-11-19T17:50:17,148 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:17,148 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:17,148 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/C is initiating minor compaction (all files) 2024-11-19T17:50:17,148 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/C in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:17,149 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/7535ed8f75cd47e2952707b9ca2e8a38, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/6c3c2979f0ce41c98157467d926fdae5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d4683d10f9f54eaaa689eba6d4305805, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/320d857fffcf4b309b58c5931bae45a3] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=47.0 K 2024-11-19T17:50:17,149 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 7535ed8f75cd47e2952707b9ca2e8a38, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732038612333 2024-11-19T17:50:17,149 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c3c2979f0ce41c98157467d926fdae5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732038613539 2024-11-19T17:50:17,149 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d4683d10f9f54eaaa689eba6d4305805, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732038613624 2024-11-19T17:50:17,150 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 320d857fffcf4b309b58c5931bae45a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038615483 2024-11-19T17:50:17,151 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/A of 65f2046e0a9c76be2cfb08c4db762a3f into 38c9797e72e54beea565730e5fe22a7c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:17,151 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:17,151 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/A, priority=12, startTime=1732038616727; duration=0sec 2024-11-19T17:50:17,151 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:17,151 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:A 2024-11-19T17:50:17,157 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:17,158 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/9d8a0560f1234f8f82ca449f71eae8e5 is 50, key is test_row_0/C:col10/1732038615484/Put/seqid=0 2024-11-19T17:50:17,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742305_1481 (size=12241) 2024-11-19T17:50:17,268 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/b3fbdc5809dd4b7f82840047e0431d92 2024-11-19T17:50:17,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/b962ddf0ae8946f099bda6e6cb4ddd74 is 50, key is test_row_0/B:col10/1732038615485/Put/seqid=0 2024-11-19T17:50:17,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742306_1482 (size=12001) 2024-11-19T17:50:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:50:17,569 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/9d8a0560f1234f8f82ca449f71eae8e5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9d8a0560f1234f8f82ca449f71eae8e5 2024-11-19T17:50:17,573 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/C of 65f2046e0a9c76be2cfb08c4db762a3f into 9d8a0560f1234f8f82ca449f71eae8e5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:17,573 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:17,573 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/C, priority=12, startTime=1732038616727; duration=0sec 2024-11-19T17:50:17,573 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:17,573 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:C 2024-11-19T17:50:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:17,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:17,678 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/b962ddf0ae8946f099bda6e6cb4ddd74 2024-11-19T17:50:17,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/4274b985fcd740a2b54d9b8ad7c16aac is 50, key is test_row_0/C:col10/1732038615485/Put/seqid=0 2024-11-19T17:50:17,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742307_1483 (size=12001) 2024-11-19T17:50:17,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038677689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038677690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038677691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038677695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038677698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038677808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038677808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038677808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038677808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:17,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038677808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038678018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038678018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038678018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038678020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038678020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,087 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/4274b985fcd740a2b54d9b8ad7c16aac 2024-11-19T17:50:18,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/b3fbdc5809dd4b7f82840047e0431d92 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/b3fbdc5809dd4b7f82840047e0431d92 2024-11-19T17:50:18,100 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/b3fbdc5809dd4b7f82840047e0431d92, entries=150, sequenceid=125, filesize=11.7 K 2024-11-19T17:50:18,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/b962ddf0ae8946f099bda6e6cb4ddd74 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/b962ddf0ae8946f099bda6e6cb4ddd74 2024-11-19T17:50:18,103 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/b962ddf0ae8946f099bda6e6cb4ddd74, entries=150, sequenceid=125, filesize=11.7 K 2024-11-19T17:50:18,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/4274b985fcd740a2b54d9b8ad7c16aac as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/4274b985fcd740a2b54d9b8ad7c16aac 2024-11-19T17:50:18,107 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/4274b985fcd740a2b54d9b8ad7c16aac, entries=150, sequenceid=125, filesize=11.7 K 2024-11-19T17:50:18,108 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 65f2046e0a9c76be2cfb08c4db762a3f in 1247ms, sequenceid=125, compaction requested=false 2024-11-19T17:50:18,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:18,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T17:50:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T17:50:18,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T17:50:18,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-11-19T17:50:18,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.8660 sec 2024-11-19T17:50:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:18,335 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:18,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T17:50:18,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/1299eb8f806f4ecdbd0d1c0cc936af62 is 50, key is test_row_0/A:col10/1732038617694/Put/seqid=0 2024-11-19T17:50:18,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038678339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038678340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038678343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038678350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742308_1484 (size=14541) 2024-11-19T17:50:18,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/1299eb8f806f4ecdbd0d1c0cc936af62 2024-11-19T17:50:18,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5c718c7ad70a47c69700a3e76fcf6e64 is 50, key is test_row_0/B:col10/1732038617694/Put/seqid=0 2024-11-19T17:50:18,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038678352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742309_1485 (size=12151) 2024-11-19T17:50:18,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5c718c7ad70a47c69700a3e76fcf6e64 2024-11-19T17:50:18,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/f9b854e1513d41debf665482185e5ac2 is 50, key is test_row_0/C:col10/1732038617694/Put/seqid=0 2024-11-19T17:50:18,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742310_1486 (size=12151) 2024-11-19T17:50:18,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/f9b854e1513d41debf665482185e5ac2 2024-11-19T17:50:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:50:18,399 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:18,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T17:50:18,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:50:18,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:18,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:18,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/1299eb8f806f4ecdbd0d1c0cc936af62 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/1299eb8f806f4ecdbd0d1c0cc936af62 2024-11-19T17:50:18,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/1299eb8f806f4ecdbd0d1c0cc936af62, entries=200, sequenceid=155, filesize=14.2 K 2024-11-19T17:50:18,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5c718c7ad70a47c69700a3e76fcf6e64 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5c718c7ad70a47c69700a3e76fcf6e64 2024-11-19T17:50:18,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5c718c7ad70a47c69700a3e76fcf6e64, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T17:50:18,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/f9b854e1513d41debf665482185e5ac2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/f9b854e1513d41debf665482185e5ac2 2024-11-19T17:50:18,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/f9b854e1513d41debf665482185e5ac2, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T17:50:18,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 65f2046e0a9c76be2cfb08c4db762a3f in 96ms, sequenceid=155, compaction requested=true 2024-11-19T17:50:18,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:18,431 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:18,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:18,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:18,431 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:18,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:18,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:18,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:18,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:18,434 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:18,434 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:18,435 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/A is initiating minor compaction (all files) 2024-11-19T17:50:18,435 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/A in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,435 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/38c9797e72e54beea565730e5fe22a7c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/b3fbdc5809dd4b7f82840047e0431d92, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/1299eb8f806f4ecdbd0d1c0cc936af62] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=37.9 K 2024-11-19T17:50:18,435 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/B is initiating minor compaction (all files) 2024-11-19T17:50:18,435 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/B in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,435 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/f19191260672431cb05e1116f9cbab07, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/b962ddf0ae8946f099bda6e6cb4ddd74, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5c718c7ad70a47c69700a3e76fcf6e64] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=35.5 K 2024-11-19T17:50:18,435 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c9797e72e54beea565730e5fe22a7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038615483 2024-11-19T17:50:18,435 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting f19191260672431cb05e1116f9cbab07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038615483 2024-11-19T17:50:18,435 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3fbdc5809dd4b7f82840047e0431d92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732038615485 2024-11-19T17:50:18,436 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b962ddf0ae8946f099bda6e6cb4ddd74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732038615485 2024-11-19T17:50:18,436 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1299eb8f806f4ecdbd0d1c0cc936af62, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038617689 2024-11-19T17:50:18,438 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c718c7ad70a47c69700a3e76fcf6e64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038617694 2024-11-19T17:50:18,447 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:18,447 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/51388e89fc2c4eed940c2466c218d87a is 50, key is test_row_0/B:col10/1732038617694/Put/seqid=0 2024-11-19T17:50:18,450 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:18,450 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/30c84fa4f9ed42a7ac0415a555ad7723 is 50, key is test_row_0/A:col10/1732038617694/Put/seqid=0 2024-11-19T17:50:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:18,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:18,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:50:18,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742312_1488 (size=12493) 2024-11-19T17:50:18,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742311_1487 (size=12493) 2024-11-19T17:50:18,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/124c73cff8624cae9b88ec1a0b0cadfb is 50, key is test_row_0/A:col10/1732038618461/Put/seqid=0 2024-11-19T17:50:18,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742313_1489 (size=14541) 2024-11-19T17:50:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:50:18,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/124c73cff8624cae9b88ec1a0b0cadfb 2024-11-19T17:50:18,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038678501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038678506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/111211513f0249028e2d924dc6094479 is 50, key is test_row_0/B:col10/1732038618461/Put/seqid=0 2024-11-19T17:50:18,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038678515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038678516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742314_1490 (size=12151) 2024-11-19T17:50:18,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038678516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:50:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038678618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038678618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038678628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038678628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038678638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:50:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:50:18,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038678828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038678828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038678838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038678838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038678849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:18,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:50:18,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:18,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:18,878 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/30c84fa4f9ed42a7ac0415a555ad7723 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/30c84fa4f9ed42a7ac0415a555ad7723 2024-11-19T17:50:18,882 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/A of 65f2046e0a9c76be2cfb08c4db762a3f into 30c84fa4f9ed42a7ac0415a555ad7723(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:18,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:18,882 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/A, priority=13, startTime=1732038618431; duration=0sec 2024-11-19T17:50:18,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:18,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:A 2024-11-19T17:50:18,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:18,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:18,882 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/C is initiating minor compaction (all files) 2024-11-19T17:50:18,883 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/C in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:18,883 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9d8a0560f1234f8f82ca449f71eae8e5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/4274b985fcd740a2b54d9b8ad7c16aac, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/f9b854e1513d41debf665482185e5ac2] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=35.5 K 2024-11-19T17:50:18,883 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d8a0560f1234f8f82ca449f71eae8e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732038615483 2024-11-19T17:50:18,883 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4274b985fcd740a2b54d9b8ad7c16aac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732038615485 2024-11-19T17:50:18,883 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9b854e1513d41debf665482185e5ac2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038617694 2024-11-19T17:50:18,888 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#C#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:18,889 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/24dced735dc74db7a414a382d83e1a92 is 50, key is test_row_0/C:col10/1732038617694/Put/seqid=0 2024-11-19T17:50:18,894 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/51388e89fc2c4eed940c2466c218d87a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/51388e89fc2c4eed940c2466c218d87a 2024-11-19T17:50:18,898 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/B of 65f2046e0a9c76be2cfb08c4db762a3f into 51388e89fc2c4eed940c2466c218d87a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:18,898 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:18,898 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/B, priority=13, startTime=1732038618431; duration=0sec 2024-11-19T17:50:18,898 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:18,898 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:B 2024-11-19T17:50:18,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742315_1491 (size=12493) 2024-11-19T17:50:18,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/111211513f0249028e2d924dc6094479 2024-11-19T17:50:18,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/892f3c51a5ea4ab0b05700371cc004df is 50, key is test_row_0/C:col10/1732038618461/Put/seqid=0 2024-11-19T17:50:18,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742316_1492 (size=12151) 2024-11-19T17:50:19,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:50:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:19,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:50:19,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038679139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038679139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038679148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038679148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038679159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:50:19,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:19,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:19,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:19,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,305 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/24dced735dc74db7a414a382d83e1a92 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/24dced735dc74db7a414a382d83e1a92 2024-11-19T17:50:19,308 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/C of 65f2046e0a9c76be2cfb08c4db762a3f into 24dced735dc74db7a414a382d83e1a92(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:19,308 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:19,308 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/C, priority=13, startTime=1732038618431; duration=0sec 2024-11-19T17:50:19,308 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:19,309 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:C 2024-11-19T17:50:19,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:50:19,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:19,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:19,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:19,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:19,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/892f3c51a5ea4ab0b05700371cc004df 2024-11-19T17:50:19,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/124c73cff8624cae9b88ec1a0b0cadfb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/124c73cff8624cae9b88ec1a0b0cadfb 2024-11-19T17:50:19,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/124c73cff8624cae9b88ec1a0b0cadfb, entries=200, sequenceid=166, filesize=14.2 K 2024-11-19T17:50:19,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/111211513f0249028e2d924dc6094479 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/111211513f0249028e2d924dc6094479 2024-11-19T17:50:19,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/111211513f0249028e2d924dc6094479, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T17:50:19,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/892f3c51a5ea4ab0b05700371cc004df as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/892f3c51a5ea4ab0b05700371cc004df 2024-11-19T17:50:19,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/892f3c51a5ea4ab0b05700371cc004df, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T17:50:19,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 65f2046e0a9c76be2cfb08c4db762a3f in 890ms, sequenceid=166, compaction requested=false 2024-11-19T17:50:19,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:19,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:50:19,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:19,471 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:50:19,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/711572d509334fae8d4f81a82d842f44 is 50, key is test_row_0/A:col10/1732038618490/Put/seqid=0 2024-11-19T17:50:19,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742317_1493 (size=12151) 2024-11-19T17:50:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:50:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:19,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:19,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038679667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038679668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038679669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038679676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038679676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038679779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038679779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038679788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038679788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,878 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/711572d509334fae8d4f81a82d842f44 2024-11-19T17:50:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/d81c471e02ae49c4ba57f746e2a36e57 is 50, key is test_row_0/B:col10/1732038618490/Put/seqid=0 2024-11-19T17:50:19,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742318_1494 (size=12151) 2024-11-19T17:50:19,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038679989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038679989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038679998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038679998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,296 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/d81c471e02ae49c4ba57f746e2a36e57 2024-11-19T17:50:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/8c4241cb27af4c87a09ab2ef03348f45 is 50, key is test_row_0/C:col10/1732038618490/Put/seqid=0 2024-11-19T17:50:20,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038680299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038680299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742319_1495 (size=12151) 2024-11-19T17:50:20,310 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/8c4241cb27af4c87a09ab2ef03348f45 2024-11-19T17:50:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/711572d509334fae8d4f81a82d842f44 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711572d509334fae8d4f81a82d842f44 2024-11-19T17:50:20,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038680310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,318 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711572d509334fae8d4f81a82d842f44, entries=150, sequenceid=194, filesize=11.9 K 2024-11-19T17:50:20,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/d81c471e02ae49c4ba57f746e2a36e57 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d81c471e02ae49c4ba57f746e2a36e57 2024-11-19T17:50:20,321 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d81c471e02ae49c4ba57f746e2a36e57, entries=150, sequenceid=194, filesize=11.9 K 2024-11-19T17:50:20,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/8c4241cb27af4c87a09ab2ef03348f45 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/8c4241cb27af4c87a09ab2ef03348f45 2024-11-19T17:50:20,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038680318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,329 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/8c4241cb27af4c87a09ab2ef03348f45, entries=150, sequenceid=194, filesize=11.9 K 2024-11-19T17:50:20,329 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 65f2046e0a9c76be2cfb08c4db762a3f in 858ms, sequenceid=194, compaction requested=true 2024-11-19T17:50:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T17:50:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T17:50:20,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T17:50:20,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9260 sec 2024-11-19T17:50:20,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.9330 sec 2024-11-19T17:50:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:50:20,549 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:20,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-19T17:50:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:50:20,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:20,553 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:20,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:50:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:20,693 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:20,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:50:20,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ef24974c9c414f2ebd8a1d57e0d7c9e4 is 50, key is test_row_1/A:col10/1732038619675/Put/seqid=0 2024-11-19T17:50:20,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:50:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:20,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:20,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742320_1496 (size=12147) 2024-11-19T17:50:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038680818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038680820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038680822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038680828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038680839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:50:20,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:20,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:20,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:20,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:20,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:50:20,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038680929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:20,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038680939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:50:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ef24974c9c414f2ebd8a1d57e0d7c9e4 2024-11-19T17:50:21,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5048d7214a694d4ab4c1e84d0da46f87 is 50, key is test_row_1/B:col10/1732038619675/Put/seqid=0 2024-11-19T17:50:21,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742321_1497 (size=9757) 2024-11-19T17:50:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038681138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038681149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:50:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:50:21,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:50:21,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:21,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038681458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038681459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:50:21,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:21,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:21,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5048d7214a694d4ab4c1e84d0da46f87 2024-11-19T17:50:21,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/fef18ae77c124d75a01d81c7c8a2e0b2 is 50, key is test_row_1/C:col10/1732038619675/Put/seqid=0 2024-11-19T17:50:21,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742322_1498 (size=9757) 2024-11-19T17:50:21,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/fef18ae77c124d75a01d81c7c8a2e0b2 2024-11-19T17:50:21,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ef24974c9c414f2ebd8a1d57e0d7c9e4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef24974c9c414f2ebd8a1d57e0d7c9e4 2024-11-19T17:50:21,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef24974c9c414f2ebd8a1d57e0d7c9e4, entries=150, sequenceid=207, filesize=11.9 K 2024-11-19T17:50:21,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5048d7214a694d4ab4c1e84d0da46f87 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5048d7214a694d4ab4c1e84d0da46f87 2024-11-19T17:50:21,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5048d7214a694d4ab4c1e84d0da46f87, entries=100, sequenceid=207, filesize=9.5 K 2024-11-19T17:50:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/fef18ae77c124d75a01d81c7c8a2e0b2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fef18ae77c124d75a01d81c7c8a2e0b2 2024-11-19T17:50:21,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fef18ae77c124d75a01d81c7c8a2e0b2, entries=100, sequenceid=207, filesize=9.5 K 2024-11-19T17:50:21,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 65f2046e0a9c76be2cfb08c4db762a3f in 855ms, sequenceid=207, compaction requested=true 2024-11-19T17:50:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:21,548 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:21,548 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:21,549 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:21,549 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/A is initiating minor compaction (all files) 2024-11-19T17:50:21,549 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/A in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,549 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/30c84fa4f9ed42a7ac0415a555ad7723, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/124c73cff8624cae9b88ec1a0b0cadfb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711572d509334fae8d4f81a82d842f44, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef24974c9c414f2ebd8a1d57e0d7c9e4] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=50.1 K 2024-11-19T17:50:21,549 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:21,549 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/B is initiating minor compaction (all files) 2024-11-19T17:50:21,549 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/B in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,549 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/51388e89fc2c4eed940c2466c218d87a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/111211513f0249028e2d924dc6094479, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d81c471e02ae49c4ba57f746e2a36e57, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5048d7214a694d4ab4c1e84d0da46f87] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=45.5 K 2024-11-19T17:50:21,550 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30c84fa4f9ed42a7ac0415a555ad7723, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038617694 2024-11-19T17:50:21,550 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 51388e89fc2c4eed940c2466c218d87a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038617694 2024-11-19T17:50:21,550 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 124c73cff8624cae9b88ec1a0b0cadfb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732038618341 2024-11-19T17:50:21,550 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 111211513f0249028e2d924dc6094479, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732038618348 2024-11-19T17:50:21,550 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 711572d509334fae8d4f81a82d842f44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732038618490 2024-11-19T17:50:21,550 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d81c471e02ae49c4ba57f746e2a36e57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732038618490 2024-11-19T17:50:21,550 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef24974c9c414f2ebd8a1d57e0d7c9e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732038619674 2024-11-19T17:50:21,550 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 5048d7214a694d4ab4c1e84d0da46f87, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732038619675 2024-11-19T17:50:21,564 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#A#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:21,564 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/410145f46e4c4891954f102b1ceb30b0 is 50, key is test_row_0/A:col10/1732038618490/Put/seqid=0 2024-11-19T17:50:21,564 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#B#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:21,565 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/6844a7545a554259a042f21a3a58eaff is 50, key is test_row_0/B:col10/1732038618490/Put/seqid=0 2024-11-19T17:50:21,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742323_1499 (size=12629) 2024-11-19T17:50:21,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742324_1500 (size=12629) 2024-11-19T17:50:21,574 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/410145f46e4c4891954f102b1ceb30b0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/410145f46e4c4891954f102b1ceb30b0 2024-11-19T17:50:21,578 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/A of 65f2046e0a9c76be2cfb08c4db762a3f into 410145f46e4c4891954f102b1ceb30b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:21,578 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:21,578 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/A, priority=12, startTime=1732038621548; duration=0sec 2024-11-19T17:50:21,578 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:21,578 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:A 2024-11-19T17:50:21,578 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:21,579 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:21,579 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/C is initiating minor compaction (all files) 2024-11-19T17:50:21,579 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/C in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,579 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/24dced735dc74db7a414a382d83e1a92, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/892f3c51a5ea4ab0b05700371cc004df, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/8c4241cb27af4c87a09ab2ef03348f45, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fef18ae77c124d75a01d81c7c8a2e0b2] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=45.5 K 2024-11-19T17:50:21,579 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24dced735dc74db7a414a382d83e1a92, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732038617694 2024-11-19T17:50:21,579 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 892f3c51a5ea4ab0b05700371cc004df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732038618348 2024-11-19T17:50:21,580 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c4241cb27af4c87a09ab2ef03348f45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732038618490 2024-11-19T17:50:21,580 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting fef18ae77c124d75a01d81c7c8a2e0b2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732038619675 2024-11-19T17:50:21,586 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:21,587 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/93c6fd867d024058abc10cadbb6afbe6 is 50, key is test_row_0/C:col10/1732038618490/Put/seqid=0 2024-11-19T17:50:21,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742325_1501 (size=12629) 2024-11-19T17:50:21,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:50:21,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:21,624 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T17:50:21,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ec51db0a35d14423b8085965311694f1 is 50, key is test_row_0/A:col10/1732038620821/Put/seqid=0 2024-11-19T17:50:21,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742326_1502 (size=12151) 2024-11-19T17:50:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:50:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:21,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038681859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038681866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038681867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,976 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/6844a7545a554259a042f21a3a58eaff as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/6844a7545a554259a042f21a3a58eaff 2024-11-19T17:50:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038681969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,980 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/B of 65f2046e0a9c76be2cfb08c4db762a3f into 6844a7545a554259a042f21a3a58eaff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:21,980 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:21,980 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/B, priority=12, startTime=1732038621548; duration=0sec 2024-11-19T17:50:21,980 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:21,980 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:B 2024-11-19T17:50:21,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038681978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038681984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038681984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038681989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:21,997 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/93c6fd867d024058abc10cadbb6afbe6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/93c6fd867d024058abc10cadbb6afbe6 2024-11-19T17:50:22,003 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/C of 65f2046e0a9c76be2cfb08c4db762a3f into 93c6fd867d024058abc10cadbb6afbe6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:22,003 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:22,003 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/C, priority=12, startTime=1732038621548; duration=0sec 2024-11-19T17:50:22,003 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:22,003 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:C 2024-11-19T17:50:22,032 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ec51db0a35d14423b8085965311694f1 2024-11-19T17:50:22,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/37c44a41f5154284927c22c7a35e2b37 is 50, key is test_row_0/B:col10/1732038620821/Put/seqid=0 2024-11-19T17:50:22,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742327_1503 (size=12151) 2024-11-19T17:50:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038682198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038682198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038682199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:22,442 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/37c44a41f5154284927c22c7a35e2b37 2024-11-19T17:50:22,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/a64e6fc3d89c484aba4958dc25db7f16 is 50, key is test_row_0/C:col10/1732038620821/Put/seqid=0 2024-11-19T17:50:22,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742328_1504 (size=12151) 2024-11-19T17:50:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038682518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038682518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038682518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:50:22,850 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/a64e6fc3d89c484aba4958dc25db7f16 2024-11-19T17:50:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/ec51db0a35d14423b8085965311694f1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ec51db0a35d14423b8085965311694f1 2024-11-19T17:50:22,856 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ec51db0a35d14423b8085965311694f1, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T17:50:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/37c44a41f5154284927c22c7a35e2b37 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/37c44a41f5154284927c22c7a35e2b37 2024-11-19T17:50:22,859 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/37c44a41f5154284927c22c7a35e2b37, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T17:50:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/a64e6fc3d89c484aba4958dc25db7f16 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a64e6fc3d89c484aba4958dc25db7f16 2024-11-19T17:50:22,863 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a64e6fc3d89c484aba4958dc25db7f16, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T17:50:22,863 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 65f2046e0a9c76be2cfb08c4db762a3f in 1239ms, sequenceid=231, compaction requested=false 2024-11-19T17:50:22,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-19T17:50:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-19T17:50:22,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T17:50:22,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3120 sec 2024-11-19T17:50:22,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.3170 sec 2024-11-19T17:50:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:22,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:22,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T17:50:22,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/4757d12b37944edda491bfc59ac8d8cd is 50, key is test_row_0/A:col10/1732038622992/Put/seqid=0 2024-11-19T17:50:22,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742329_1505 (size=14541) 2024-11-19T17:50:23,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038683043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038683051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038683052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038683053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038683053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038683158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038683158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038683169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038683169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038683169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038683368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038683369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038683379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038683379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038683379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/4757d12b37944edda491bfc59ac8d8cd 2024-11-19T17:50:23,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/95e396e6c14a4594b20b4c8ba7e2ca67 is 50, key is test_row_0/B:col10/1732038622992/Put/seqid=0 2024-11-19T17:50:23,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742330_1506 (size=12151) 2024-11-19T17:50:23,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038683678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038683678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038683689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038683689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038683698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:23,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/95e396e6c14a4594b20b4c8ba7e2ca67 2024-11-19T17:50:23,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/0cbded8c938d4c1786a5aa52e8c25b44 is 50, key is test_row_0/C:col10/1732038622992/Put/seqid=0 2024-11-19T17:50:23,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742331_1507 (size=12151) 2024-11-19T17:50:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038684188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038684188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038684199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038684199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:24,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038684209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:24,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/0cbded8c938d4c1786a5aa52e8c25b44 2024-11-19T17:50:24,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/4757d12b37944edda491bfc59ac8d8cd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4757d12b37944edda491bfc59ac8d8cd 2024-11-19T17:50:24,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4757d12b37944edda491bfc59ac8d8cd, entries=200, sequenceid=247, filesize=14.2 K 2024-11-19T17:50:24,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/95e396e6c14a4594b20b4c8ba7e2ca67 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/95e396e6c14a4594b20b4c8ba7e2ca67 2024-11-19T17:50:24,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/95e396e6c14a4594b20b4c8ba7e2ca67, entries=150, sequenceid=247, filesize=11.9 K 2024-11-19T17:50:24,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/0cbded8c938d4c1786a5aa52e8c25b44 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/0cbded8c938d4c1786a5aa52e8c25b44 2024-11-19T17:50:24,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/0cbded8c938d4c1786a5aa52e8c25b44, entries=150, sequenceid=247, filesize=11.9 K 2024-11-19T17:50:24,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 65f2046e0a9c76be2cfb08c4db762a3f in 1243ms, sequenceid=247, compaction requested=true 2024-11-19T17:50:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:24,237 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:24,237 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:24,237 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:24,237 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:24,237 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/B is initiating minor compaction (all files) 2024-11-19T17:50:24,237 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/A is initiating minor compaction (all files) 2024-11-19T17:50:24,237 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/A in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:24,237 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/B in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:24,237 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/6844a7545a554259a042f21a3a58eaff, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/37c44a41f5154284927c22c7a35e2b37, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/95e396e6c14a4594b20b4c8ba7e2ca67] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=36.1 K 2024-11-19T17:50:24,237 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/410145f46e4c4891954f102b1ceb30b0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ec51db0a35d14423b8085965311694f1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4757d12b37944edda491bfc59ac8d8cd] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=38.4 K 2024-11-19T17:50:24,238 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 410145f46e4c4891954f102b1ceb30b0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732038618490 2024-11-19T17:50:24,238 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 6844a7545a554259a042f21a3a58eaff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732038618490 2024-11-19T17:50:24,238 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec51db0a35d14423b8085965311694f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732038620800 2024-11-19T17:50:24,238 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 37c44a41f5154284927c22c7a35e2b37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732038620800 2024-11-19T17:50:24,238 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e396e6c14a4594b20b4c8ba7e2ca67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732038621848 2024-11-19T17:50:24,238 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4757d12b37944edda491bfc59ac8d8cd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732038621848 2024-11-19T17:50:24,244 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#A#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:24,244 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/82f6386047144e45b1971aa40c09c1c6 is 50, key is test_row_0/A:col10/1732038622992/Put/seqid=0 2024-11-19T17:50:24,244 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:24,245 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/1a958bada20f4bcebd956b876f249e2f is 50, key is test_row_0/B:col10/1732038622992/Put/seqid=0 2024-11-19T17:50:24,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742332_1508 (size=12731) 2024-11-19T17:50:24,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742333_1509 (size=12731) 2024-11-19T17:50:24,663 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/82f6386047144e45b1971aa40c09c1c6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/82f6386047144e45b1971aa40c09c1c6 2024-11-19T17:50:24,663 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/1a958bada20f4bcebd956b876f249e2f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/1a958bada20f4bcebd956b876f249e2f 2024-11-19T17:50:24,667 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/A of 65f2046e0a9c76be2cfb08c4db762a3f into 82f6386047144e45b1971aa40c09c1c6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:24,667 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:24,667 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/A, priority=13, startTime=1732038624236; duration=0sec 2024-11-19T17:50:24,668 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:24,668 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:A 2024-11-19T17:50:24,668 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:24,668 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/B of 65f2046e0a9c76be2cfb08c4db762a3f into 1a958bada20f4bcebd956b876f249e2f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:24,668 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:24,668 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/B, priority=13, startTime=1732038624237; duration=0sec 2024-11-19T17:50:24,668 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:24,668 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:B 2024-11-19T17:50:24,668 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:24,668 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/C is initiating minor compaction (all files) 2024-11-19T17:50:24,668 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/C in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:24,668 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/93c6fd867d024058abc10cadbb6afbe6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a64e6fc3d89c484aba4958dc25db7f16, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/0cbded8c938d4c1786a5aa52e8c25b44] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=36.1 K 2024-11-19T17:50:24,669 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c6fd867d024058abc10cadbb6afbe6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732038618490 2024-11-19T17:50:24,669 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting a64e6fc3d89c484aba4958dc25db7f16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732038620800 2024-11-19T17:50:24,669 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cbded8c938d4c1786a5aa52e8c25b44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732038621848 2024-11-19T17:50:24,674 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:24,674 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/b964c8723a234842813dc47d2fa8f0a8 is 50, key is test_row_0/C:col10/1732038622992/Put/seqid=0 2024-11-19T17:50:24,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742334_1510 (size=12731) 2024-11-19T17:50:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:50:24,719 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:24,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-19T17:50:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:50:24,721 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:24,721 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:24,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:50:24,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T17:50:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:24,874 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T17:50:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/7f044e6e9e464afcb2c416bd80f064eb is 50, key is test_row_0/A:col10/1732038623052/Put/seqid=0 2024-11-19T17:50:24,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742335_1511 (size=12301) 2024-11-19T17:50:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:50:25,081 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/b964c8723a234842813dc47d2fa8f0a8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/b964c8723a234842813dc47d2fa8f0a8 2024-11-19T17:50:25,084 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/C of 65f2046e0a9c76be2cfb08c4db762a3f into b964c8723a234842813dc47d2fa8f0a8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:25,084 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:25,084 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/C, priority=13, startTime=1732038624237; duration=0sec 2024-11-19T17:50:25,084 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:25,084 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:C 2024-11-19T17:50:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:25,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:25,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038685222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038685222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038685223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038685226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038685228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,237 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 65f2046e0a9c76be2cfb08c4db762a3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,282 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/7f044e6e9e464afcb2c416bd80f064eb 2024-11-19T17:50:25,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5fe00cf1a03d418fbbb5ab06d7aa5105 is 50, key is test_row_0/B:col10/1732038623052/Put/seqid=0 2024-11-19T17:50:25,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742336_1512 (size=12301) 2024-11-19T17:50:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038685329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038685329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038685338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038685338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:50:25,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038685539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038685539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038685548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038685548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,700 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5fe00cf1a03d418fbbb5ab06d7aa5105 2024-11-19T17:50:25,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/96b9f606021342d29223a148a9e4a6a5 is 50, key is test_row_0/C:col10/1732038623052/Put/seqid=0 2024-11-19T17:50:25,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742337_1513 (size=12301) 2024-11-19T17:50:25,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038685849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038685849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:50:25,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038685859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038685859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,111 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/96b9f606021342d29223a148a9e4a6a5 2024-11-19T17:50:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/7f044e6e9e464afcb2c416bd80f064eb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/7f044e6e9e464afcb2c416bd80f064eb 2024-11-19T17:50:26,117 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/7f044e6e9e464afcb2c416bd80f064eb, entries=150, sequenceid=271, filesize=12.0 K 2024-11-19T17:50:26,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/5fe00cf1a03d418fbbb5ab06d7aa5105 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5fe00cf1a03d418fbbb5ab06d7aa5105 2024-11-19T17:50:26,120 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5fe00cf1a03d418fbbb5ab06d7aa5105, entries=150, sequenceid=271, filesize=12.0 K 2024-11-19T17:50:26,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/96b9f606021342d29223a148a9e4a6a5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/96b9f606021342d29223a148a9e4a6a5 2024-11-19T17:50:26,124 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/96b9f606021342d29223a148a9e4a6a5, entries=150, sequenceid=271, filesize=12.0 K 2024-11-19T17:50:26,124 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 65f2046e0a9c76be2cfb08c4db762a3f in 1251ms, sequenceid=271, compaction requested=false 2024-11-19T17:50:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-19T17:50:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-19T17:50:26,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T17:50:26,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-19T17:50:26,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4090 sec 2024-11-19T17:50:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:26,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T17:50:26,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/8e6d0b2f30fa4cddb87a7c56d2babce0 is 50, key is test_row_0/A:col10/1732038625224/Put/seqid=0 2024-11-19T17:50:26,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742338_1514 (size=14741) 2024-11-19T17:50:26,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038686406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038686407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038686408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038686409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038686518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038686518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038686529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038686529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038686728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038686728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038686738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038686739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:26,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/8e6d0b2f30fa4cddb87a7c56d2babce0 2024-11-19T17:50:26,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c6f59f4b940143b7bebd5845d4a1f117 is 50, key is test_row_0/B:col10/1732038625224/Put/seqid=0 2024-11-19T17:50:26,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742339_1515 (size=12301) 2024-11-19T17:50:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:50:26,869 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:26,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-19T17:50:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:50:26,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:26,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:26,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:50:27,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:50:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038687038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038687038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038687048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038687059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:50:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c6f59f4b940143b7bebd5845d4a1f117 2024-11-19T17:50:27,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/bd5b15837dba49e4b7cc551c609ecf99 is 50, key is test_row_0/C:col10/1732038625224/Put/seqid=0 2024-11-19T17:50:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:50:27,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742340_1516 (size=12301) 2024-11-19T17:50:27,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038687258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,264 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in 65f2046e0a9c76be2cfb08c4db762a3f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:50:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:50:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:50:27,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038687549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038687549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038687559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038687578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:27,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:27,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/bd5b15837dba49e4b7cc551c609ecf99 2024-11-19T17:50:27,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/8e6d0b2f30fa4cddb87a7c56d2babce0 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/8e6d0b2f30fa4cddb87a7c56d2babce0 2024-11-19T17:50:27,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/8e6d0b2f30fa4cddb87a7c56d2babce0, entries=200, sequenceid=287, filesize=14.4 K 2024-11-19T17:50:27,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c6f59f4b940143b7bebd5845d4a1f117 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c6f59f4b940143b7bebd5845d4a1f117 2024-11-19T17:50:27,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c6f59f4b940143b7bebd5845d4a1f117, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T17:50:27,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/bd5b15837dba49e4b7cc551c609ecf99 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/bd5b15837dba49e4b7cc551c609ecf99 2024-11-19T17:50:27,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/bd5b15837dba49e4b7cc551c609ecf99, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T17:50:27,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 65f2046e0a9c76be2cfb08c4db762a3f in 1244ms, sequenceid=287, compaction requested=true 2024-11-19T17:50:27,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:27,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:27,604 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:27,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:27,605 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:27,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:27,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:27,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:27,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:27,605 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:27,605 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:27,605 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/A is initiating minor compaction (all files) 2024-11-19T17:50:27,605 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/B is initiating minor compaction (all files) 2024-11-19T17:50:27,606 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/A in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,606 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/B in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,606 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/1a958bada20f4bcebd956b876f249e2f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5fe00cf1a03d418fbbb5ab06d7aa5105, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c6f59f4b940143b7bebd5845d4a1f117] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=36.5 K 2024-11-19T17:50:27,606 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/82f6386047144e45b1971aa40c09c1c6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/7f044e6e9e464afcb2c416bd80f064eb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/8e6d0b2f30fa4cddb87a7c56d2babce0] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=38.8 K 2024-11-19T17:50:27,606 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82f6386047144e45b1971aa40c09c1c6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732038621848 2024-11-19T17:50:27,606 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a958bada20f4bcebd956b876f249e2f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732038621848 2024-11-19T17:50:27,606 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f044e6e9e464afcb2c416bd80f064eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732038623027 2024-11-19T17:50:27,606 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fe00cf1a03d418fbbb5ab06d7aa5105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732038623027 2024-11-19T17:50:27,606 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e6d0b2f30fa4cddb87a7c56d2babce0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732038625212 2024-11-19T17:50:27,606 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f59f4b940143b7bebd5845d4a1f117, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732038625222 2024-11-19T17:50:27,613 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:27,614 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/0d1d6b4ada3344e2bb77e4b69301ebe4 is 50, key is test_row_0/B:col10/1732038625224/Put/seqid=0 2024-11-19T17:50:27,620 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:27,621 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/3e54a90d8ad049238a6a6ba7b050fe89 is 50, key is test_row_0/A:col10/1732038625224/Put/seqid=0 2024-11-19T17:50:27,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742341_1517 (size=12983) 2024-11-19T17:50:27,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742342_1518 (size=12983) 2024-11-19T17:50:27,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:50:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:27,635 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T17:50:27,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/fdaffc2b083a4ee092f54cdfd962968e is 50, key is test_row_0/A:col10/1732038626396/Put/seqid=0 2024-11-19T17:50:27,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742343_1519 (size=12301) 2024-11-19T17:50:27,644 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/fdaffc2b083a4ee092f54cdfd962968e 2024-11-19T17:50:27,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c314aaef66064f4dac9989f4cf750a71 is 50, key is test_row_0/B:col10/1732038626396/Put/seqid=0 2024-11-19T17:50:27,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742344_1520 (size=12301) 2024-11-19T17:50:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:50:28,026 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/0d1d6b4ada3344e2bb77e4b69301ebe4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0d1d6b4ada3344e2bb77e4b69301ebe4 2024-11-19T17:50:28,029 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/B of 65f2046e0a9c76be2cfb08c4db762a3f into 0d1d6b4ada3344e2bb77e4b69301ebe4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:28,029 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:28,029 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/B, priority=13, startTime=1732038627605; duration=0sec 2024-11-19T17:50:28,029 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:28,029 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:B 2024-11-19T17:50:28,030 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:28,030 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:28,030 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/C is initiating minor compaction (all files) 2024-11-19T17:50:28,030 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/C in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:28,030 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/b964c8723a234842813dc47d2fa8f0a8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/96b9f606021342d29223a148a9e4a6a5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/bd5b15837dba49e4b7cc551c609ecf99] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=36.5 K 2024-11-19T17:50:28,030 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting b964c8723a234842813dc47d2fa8f0a8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732038621848 2024-11-19T17:50:28,031 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b9f606021342d29223a148a9e4a6a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732038623027 2024-11-19T17:50:28,031 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5b15837dba49e4b7cc551c609ecf99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732038625222 2024-11-19T17:50:28,032 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/3e54a90d8ad049238a6a6ba7b050fe89 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/3e54a90d8ad049238a6a6ba7b050fe89 2024-11-19T17:50:28,036 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/A of 65f2046e0a9c76be2cfb08c4db762a3f into 3e54a90d8ad049238a6a6ba7b050fe89(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:28,036 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:28,036 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/A, priority=13, startTime=1732038627604; duration=0sec 2024-11-19T17:50:28,036 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:28,037 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:A 2024-11-19T17:50:28,037 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:28,038 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/a13a4613c110407489f1f84da5e38ecf is 50, key is test_row_0/C:col10/1732038625224/Put/seqid=0 2024-11-19T17:50:28,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742345_1521 (size=12983) 2024-11-19T17:50:28,048 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/a13a4613c110407489f1f84da5e38ecf as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a13a4613c110407489f1f84da5e38ecf 2024-11-19T17:50:28,051 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/C of 65f2046e0a9c76be2cfb08c4db762a3f into a13a4613c110407489f1f84da5e38ecf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:28,051 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:28,051 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/C, priority=13, startTime=1732038627605; duration=0sec 2024-11-19T17:50:28,051 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:28,051 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:C 2024-11-19T17:50:28,058 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c314aaef66064f4dac9989f4cf750a71 2024-11-19T17:50:28,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/9f675c2224b94e7aa830a7b0a36e7fdb is 50, key is test_row_0/C:col10/1732038626396/Put/seqid=0 2024-11-19T17:50:28,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742346_1522 (size=12301) 2024-11-19T17:50:28,474 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/9f675c2224b94e7aa830a7b0a36e7fdb 2024-11-19T17:50:28,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/fdaffc2b083a4ee092f54cdfd962968e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/fdaffc2b083a4ee092f54cdfd962968e 2024-11-19T17:50:28,481 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/fdaffc2b083a4ee092f54cdfd962968e, entries=150, sequenceid=308, filesize=12.0 K 2024-11-19T17:50:28,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/c314aaef66064f4dac9989f4cf750a71 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c314aaef66064f4dac9989f4cf750a71 2024-11-19T17:50:28,484 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c314aaef66064f4dac9989f4cf750a71, entries=150, sequenceid=308, filesize=12.0 K 2024-11-19T17:50:28,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/9f675c2224b94e7aa830a7b0a36e7fdb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9f675c2224b94e7aa830a7b0a36e7fdb 2024-11-19T17:50:28,487 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9f675c2224b94e7aa830a7b0a36e7fdb, entries=150, sequenceid=308, filesize=12.0 K 2024-11-19T17:50:28,488 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 65f2046e0a9c76be2cfb08c4db762a3f in 853ms, sequenceid=308, compaction requested=false 2024-11-19T17:50:28,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:28,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:28,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-19T17:50:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-19T17:50:28,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T17:50:28,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6190 sec 2024-11-19T17:50:28,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.6220 sec 2024-11-19T17:50:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:28,578 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:28,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:50:28,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/4caf3aeb8e104275a24478f8aaa53eca is 50, key is test_row_0/A:col10/1732038628578/Put/seqid=0 2024-11-19T17:50:28,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742347_1523 (size=14737) 2024-11-19T17:50:28,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038688614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038688616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038688617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038688620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038688728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038688728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038688728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038688729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038688938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038688938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038688938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038688938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:28,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/4caf3aeb8e104275a24478f8aaa53eca 2024-11-19T17:50:28,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3a28069861824619b64db1c742cb90b8 is 50, key is test_row_0/B:col10/1732038628578/Put/seqid=0 2024-11-19T17:50:28,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742348_1524 (size=9857) 2024-11-19T17:50:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:50:29,018 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:29,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-19T17:50:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:50:29,020 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:29,021 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:29,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:50:29,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:50:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038689248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:29,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038689248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:29,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038689248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:29,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038689248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:29,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:50:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:50:29,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3a28069861824619b64db1c742cb90b8 2024-11-19T17:50:29,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/94d5f938313f41ce9c3b72c8769c1cfa is 50, key is test_row_0/C:col10/1732038628578/Put/seqid=0 2024-11-19T17:50:29,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742349_1525 (size=9857) 2024-11-19T17:50:29,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:50:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:50:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:50:29,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:50:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038689758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038689758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038689758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:29,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038689768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:29,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:50:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:29,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/94d5f938313f41ce9c3b72c8769c1cfa 2024-11-19T17:50:29,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/4caf3aeb8e104275a24478f8aaa53eca as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4caf3aeb8e104275a24478f8aaa53eca 2024-11-19T17:50:29,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4caf3aeb8e104275a24478f8aaa53eca, entries=200, sequenceid=322, filesize=14.4 K 2024-11-19T17:50:29,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/3a28069861824619b64db1c742cb90b8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3a28069861824619b64db1c742cb90b8 2024-11-19T17:50:29,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3a28069861824619b64db1c742cb90b8, entries=100, sequenceid=322, filesize=9.6 K 2024-11-19T17:50:29,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/94d5f938313f41ce9c3b72c8769c1cfa as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/94d5f938313f41ce9c3b72c8769c1cfa 2024-11-19T17:50:29,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/94d5f938313f41ce9c3b72c8769c1cfa, entries=100, sequenceid=322, filesize=9.6 K 2024-11-19T17:50:29,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 65f2046e0a9c76be2cfb08c4db762a3f in 1243ms, sequenceid=322, compaction requested=true 2024-11-19T17:50:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:29,821 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:29,821 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:29,822 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40021 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:29,822 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:29,822 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/B is initiating minor compaction (all files) 2024-11-19T17:50:29,822 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/A is initiating minor compaction (all files) 2024-11-19T17:50:29,822 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/A in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,822 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/B in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,822 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0d1d6b4ada3344e2bb77e4b69301ebe4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c314aaef66064f4dac9989f4cf750a71, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3a28069861824619b64db1c742cb90b8] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=34.3 K 2024-11-19T17:50:29,822 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/3e54a90d8ad049238a6a6ba7b050fe89, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/fdaffc2b083a4ee092f54cdfd962968e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4caf3aeb8e104275a24478f8aaa53eca] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=39.1 K 2024-11-19T17:50:29,822 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e54a90d8ad049238a6a6ba7b050fe89, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732038625222 2024-11-19T17:50:29,822 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d1d6b4ada3344e2bb77e4b69301ebe4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732038625222 2024-11-19T17:50:29,822 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c314aaef66064f4dac9989f4cf750a71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732038626396 2024-11-19T17:50:29,822 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdaffc2b083a4ee092f54cdfd962968e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732038626396 2024-11-19T17:50:29,823 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4caf3aeb8e104275a24478f8aaa53eca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732038628570 2024-11-19T17:50:29,823 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a28069861824619b64db1c742cb90b8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732038628570 2024-11-19T17:50:29,827 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:29,827 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/500fc0cf675f4700873ec8d06c07b7ee is 50, key is test_row_0/A:col10/1732038628578/Put/seqid=0 2024-11-19T17:50:29,830 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#B#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:29,830 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/64622981be3343349018018035904954 is 50, key is test_row_0/B:col10/1732038628578/Put/seqid=0 2024-11-19T17:50:29,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742351_1527 (size=13085) 2024-11-19T17:50:29,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742350_1526 (size=13085) 2024-11-19T17:50:29,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:50:29,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:29,936 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:50:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/a9d823ff0fe24f3f96cda2e7002e7fae is 50, key is test_row_0/A:col10/1732038628619/Put/seqid=0 2024-11-19T17:50:29,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742352_1528 (size=12301) 2024-11-19T17:50:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:50:30,237 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/64622981be3343349018018035904954 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/64622981be3343349018018035904954 2024-11-19T17:50:30,241 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/500fc0cf675f4700873ec8d06c07b7ee as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/500fc0cf675f4700873ec8d06c07b7ee 2024-11-19T17:50:30,241 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/B of 65f2046e0a9c76be2cfb08c4db762a3f into 64622981be3343349018018035904954(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:30,241 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:30,241 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/B, priority=13, startTime=1732038629821; duration=0sec 2024-11-19T17:50:30,241 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:30,241 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:B 2024-11-19T17:50:30,242 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:30,242 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:30,242 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/C is initiating minor compaction (all files) 2024-11-19T17:50:30,242 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/C in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:30,243 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a13a4613c110407489f1f84da5e38ecf, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9f675c2224b94e7aa830a7b0a36e7fdb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/94d5f938313f41ce9c3b72c8769c1cfa] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=34.3 K 2024-11-19T17:50:30,243 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a13a4613c110407489f1f84da5e38ecf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732038625222 2024-11-19T17:50:30,244 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f675c2224b94e7aa830a7b0a36e7fdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732038626396 2024-11-19T17:50:30,244 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d5f938313f41ce9c3b72c8769c1cfa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732038628570 2024-11-19T17:50:30,244 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/A of 65f2046e0a9c76be2cfb08c4db762a3f into 500fc0cf675f4700873ec8d06c07b7ee(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:30,244 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:30,244 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/A, priority=13, startTime=1732038629821; duration=0sec 2024-11-19T17:50:30,244 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:30,244 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:A 2024-11-19T17:50:30,250 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:30,250 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/21d18cc7a5404b9c8a228336669a881d is 50, key is test_row_0/C:col10/1732038628578/Put/seqid=0 2024-11-19T17:50:30,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742353_1529 (size=13085) 2024-11-19T17:50:30,342 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/a9d823ff0fe24f3f96cda2e7002e7fae 2024-11-19T17:50:30,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/376b87e3c56f43ff896cd3f6d369da6b is 50, key is test_row_0/B:col10/1732038628619/Put/seqid=0 2024-11-19T17:50:30,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742354_1530 (size=12301) 2024-11-19T17:50:30,659 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/21d18cc7a5404b9c8a228336669a881d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/21d18cc7a5404b9c8a228336669a881d 2024-11-19T17:50:30,662 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/C of 65f2046e0a9c76be2cfb08c4db762a3f into 21d18cc7a5404b9c8a228336669a881d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:30,662 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:30,662 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/C, priority=13, startTime=1732038629821; duration=0sec 2024-11-19T17:50:30,662 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:30,662 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:C 2024-11-19T17:50:30,752 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/376b87e3c56f43ff896cd3f6d369da6b 2024-11-19T17:50:30,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/921fa90ece354452a6936fdc2c9dcbea is 50, key is test_row_0/C:col10/1732038628619/Put/seqid=0 2024-11-19T17:50:30,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742355_1531 (size=12301) 2024-11-19T17:50:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:30,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. as already flushing 2024-11-19T17:50:30,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038690785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:30,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038690786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038690787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52010 deadline: 1732038690788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:30,797 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in 65f2046e0a9c76be2cfb08c4db762a3f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:30,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038690899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:30,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038690899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:30,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038690899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038691108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038691109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038691109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,161 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/921fa90ece354452a6936fdc2c9dcbea 2024-11-19T17:50:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/a9d823ff0fe24f3f96cda2e7002e7fae as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/a9d823ff0fe24f3f96cda2e7002e7fae 2024-11-19T17:50:31,168 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/a9d823ff0fe24f3f96cda2e7002e7fae, entries=150, sequenceid=347, filesize=12.0 K 2024-11-19T17:50:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:50:31,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/376b87e3c56f43ff896cd3f6d369da6b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/376b87e3c56f43ff896cd3f6d369da6b 2024-11-19T17:50:31,175 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/376b87e3c56f43ff896cd3f6d369da6b, entries=150, sequenceid=347, filesize=12.0 K 2024-11-19T17:50:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/921fa90ece354452a6936fdc2c9dcbea as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/921fa90ece354452a6936fdc2c9dcbea 2024-11-19T17:50:31,179 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/921fa90ece354452a6936fdc2c9dcbea, entries=150, sequenceid=347, filesize=12.0 K 2024-11-19T17:50:31,180 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 65f2046e0a9c76be2cfb08c4db762a3f in 1244ms, sequenceid=347, compaction requested=false 2024-11-19T17:50:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-19T17:50:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-19T17:50:31,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T17:50:31,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-11-19T17:50:31,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.1650 sec 2024-11-19T17:50:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:31,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:50:31,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/321710c37c7a4c31b54ad9c5fef6aad1 is 50, key is test_row_0/A:col10/1732038630786/Put/seqid=0 2024-11-19T17:50:31,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742356_1532 (size=14741) 2024-11-19T17:50:31,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038691411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038691418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038691418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038691418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038691518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/321710c37c7a4c31b54ad9c5fef6aad1 2024-11-19T17:50:31,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/146c59715bad47c6bed0cab037eb746f is 50, key is test_row_0/B:col10/1732038630786/Put/seqid=0 2024-11-19T17:50:31,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742357_1533 (size=12301) 2024-11-19T17:50:31,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038691729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51978 deadline: 1732038691928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,941 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-19T17:50:31,941 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:31,941 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,941 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-19T17:50:31,941 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:31,941 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:31,941 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51972 deadline: 1732038691939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,942 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:31,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51994 deadline: 1732038691939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:31,944 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-19T17:50:31,944 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:31,944 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,944 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,944 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:31,950 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-11-19T17:50:31,950 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:31,950 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,950 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:31,952 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-11-19T17:50:31,952 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:31,952 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,952 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:31,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:32,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52012 deadline: 1732038692039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 , the old value is region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:32,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., hostname=26d3106abcfe,36839,1732038481522, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:50:32,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/146c59715bad47c6bed0cab037eb746f 2024-11-19T17:50:32,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/fc39aaa77ddb42ae94bce2a11bd4299f is 50, key is test_row_0/C:col10/1732038630786/Put/seqid=0 2024-11-19T17:50:32,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742358_1534 (size=12301) 2024-11-19T17:50:32,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/fc39aaa77ddb42ae94bce2a11bd4299f 2024-11-19T17:50:32,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/321710c37c7a4c31b54ad9c5fef6aad1 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/321710c37c7a4c31b54ad9c5fef6aad1 2024-11-19T17:50:32,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/321710c37c7a4c31b54ad9c5fef6aad1, entries=200, sequenceid=362, filesize=14.4 K 2024-11-19T17:50:32,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/146c59715bad47c6bed0cab037eb746f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/146c59715bad47c6bed0cab037eb746f 2024-11-19T17:50:32,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/146c59715bad47c6bed0cab037eb746f, entries=150, sequenceid=362, filesize=12.0 K 2024-11-19T17:50:32,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/fc39aaa77ddb42ae94bce2a11bd4299f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fc39aaa77ddb42ae94bce2a11bd4299f 2024-11-19T17:50:32,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fc39aaa77ddb42ae94bce2a11bd4299f, entries=150, sequenceid=362, filesize=12.0 K 2024-11-19T17:50:32,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 65f2046e0a9c76be2cfb08c4db762a3f in 1262ms, sequenceid=362, compaction requested=true 2024-11-19T17:50:32,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:32,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:32,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:32,544 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:32,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:32,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:32,544 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:32,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 65f2046e0a9c76be2cfb08c4db762a3f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:32,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:32,545 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:32,545 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/A is initiating minor compaction (all files) 2024-11-19T17:50:32,545 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/A in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:32,545 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/500fc0cf675f4700873ec8d06c07b7ee, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/a9d823ff0fe24f3f96cda2e7002e7fae, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/321710c37c7a4c31b54ad9c5fef6aad1] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=39.2 K 2024-11-19T17:50:32,545 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 500fc0cf675f4700873ec8d06c07b7ee, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732038626405 2024-11-19T17:50:32,545 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:32,545 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/B is initiating minor compaction (all files) 2024-11-19T17:50:32,545 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/B in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:32,545 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d823ff0fe24f3f96cda2e7002e7fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732038628608 2024-11-19T17:50:32,545 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/64622981be3343349018018035904954, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/376b87e3c56f43ff896cd3f6d369da6b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/146c59715bad47c6bed0cab037eb746f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=36.8 K 2024-11-19T17:50:32,546 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 321710c37c7a4c31b54ad9c5fef6aad1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732038630784 2024-11-19T17:50:32,546 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 64622981be3343349018018035904954, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732038626405 2024-11-19T17:50:32,546 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 376b87e3c56f43ff896cd3f6d369da6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732038628608 2024-11-19T17:50:32,546 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 146c59715bad47c6bed0cab037eb746f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732038630784 2024-11-19T17:50:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:32,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T17:50:32,550 INFO [Thread-2048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2048. 2024-11-19T17:50:32,550 DEBUG [Thread-2048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:32,550 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,550 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:32,554 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:32,554 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#A#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:32,554 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/c60a130a9d9c46f2b935236380a69372 is 50, key is test_row_0/A:col10/1732038630786/Put/seqid=0 2024-11-19T17:50:32,554 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/0efab8e2fca44881ac84682a9e4dcf0f is 50, key is test_row_0/B:col10/1732038630786/Put/seqid=0 2024-11-19T17:50:32,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/52020425ebb548499eb28c84ec4dd683 is 50, key is test_row_0/A:col10/1732038631410/Put/seqid=0 2024-11-19T17:50:32,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742360_1536 (size=13187) 2024-11-19T17:50:32,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742359_1535 (size=13187) 2024-11-19T17:50:32,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742361_1537 (size=12301) 2024-11-19T17:50:32,810 INFO [Thread-2042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2042. 2024-11-19T17:50:32,810 DEBUG [Thread-2042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:32,810 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,810 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:32,940 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-11-19T17:50:32,940 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:32,940 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,940 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:32,950 INFO [Thread-2046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2046. 2024-11-19T17:50:32,950 DEBUG [Thread-2046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:32,950 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,950 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:32,960 INFO [Thread-2044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2044. 2024-11-19T17:50:32,960 DEBUG [Thread-2044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:32,960 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,960 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:32,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:32,965 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/0efab8e2fca44881ac84682a9e4dcf0f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0efab8e2fca44881ac84682a9e4dcf0f 2024-11-19T17:50:32,966 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/c60a130a9d9c46f2b935236380a69372 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/c60a130a9d9c46f2b935236380a69372 2024-11-19T17:50:32,969 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/B of 65f2046e0a9c76be2cfb08c4db762a3f into 0efab8e2fca44881ac84682a9e4dcf0f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:32,969 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:32,969 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/B, priority=13, startTime=1732038632544; duration=0sec 2024-11-19T17:50:32,969 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:32,969 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:B 2024-11-19T17:50:32,969 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:32,971 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:32,971 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): 65f2046e0a9c76be2cfb08c4db762a3f/C is initiating minor compaction (all files) 2024-11-19T17:50:32,971 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/A of 65f2046e0a9c76be2cfb08c4db762a3f into c60a130a9d9c46f2b935236380a69372(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:32,971 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:32,971 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 65f2046e0a9c76be2cfb08c4db762a3f/C in TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:32,971 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/A, priority=13, startTime=1732038632544; duration=0sec 2024-11-19T17:50:32,971 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/21d18cc7a5404b9c8a228336669a881d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/921fa90ece354452a6936fdc2c9dcbea, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fc39aaa77ddb42ae94bce2a11bd4299f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp, totalSize=36.8 K 2024-11-19T17:50:32,971 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:32,971 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:A 2024-11-19T17:50:32,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/52020425ebb548499eb28c84ec4dd683 2024-11-19T17:50:32,972 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d18cc7a5404b9c8a228336669a881d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732038626405 2024-11-19T17:50:32,972 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 921fa90ece354452a6936fdc2c9dcbea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732038628608 2024-11-19T17:50:32,973 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting fc39aaa77ddb42ae94bce2a11bd4299f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732038630784 2024-11-19T17:50:32,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/14fcef1603ba4ac6ae15c8d56e1314ba is 50, key is test_row_0/B:col10/1732038631410/Put/seqid=0 2024-11-19T17:50:32,979 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 65f2046e0a9c76be2cfb08c4db762a3f#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:32,980 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/d63ed31d48db449d97034b8038c24d40 is 50, key is test_row_0/C:col10/1732038630786/Put/seqid=0 2024-11-19T17:50:32,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742362_1538 (size=12301) 2024-11-19T17:50:32,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/14fcef1603ba4ac6ae15c8d56e1314ba 2024-11-19T17:50:32,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/2a22d96cd9ba407385415de2922ffafe is 50, key is test_row_0/C:col10/1732038631410/Put/seqid=0 2024-11-19T17:50:32,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742363_1539 (size=13187) 2024-11-19T17:50:32,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742364_1540 (size=12301) 2024-11-19T17:50:32,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/2a22d96cd9ba407385415de2922ffafe 2024-11-19T17:50:33,001 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/d63ed31d48db449d97034b8038c24d40 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d63ed31d48db449d97034b8038c24d40 2024-11-19T17:50:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/52020425ebb548499eb28c84ec4dd683 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/52020425ebb548499eb28c84ec4dd683 2024-11-19T17:50:33,005 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 65f2046e0a9c76be2cfb08c4db762a3f/C of 65f2046e0a9c76be2cfb08c4db762a3f into d63ed31d48db449d97034b8038c24d40(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:33,005 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:33,006 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., storeName=65f2046e0a9c76be2cfb08c4db762a3f/C, priority=13, startTime=1732038632544; duration=0sec 2024-11-19T17:50:33,006 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:33,006 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 65f2046e0a9c76be2cfb08c4db762a3f:C 2024-11-19T17:50:33,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/52020425ebb548499eb28c84ec4dd683, entries=150, sequenceid=387, filesize=12.0 K 2024-11-19T17:50:33,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/14fcef1603ba4ac6ae15c8d56e1314ba as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/14fcef1603ba4ac6ae15c8d56e1314ba 2024-11-19T17:50:33,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/14fcef1603ba4ac6ae15c8d56e1314ba, entries=150, sequenceid=387, filesize=12.0 K 2024-11-19T17:50:33,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/2a22d96cd9ba407385415de2922ffafe as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/2a22d96cd9ba407385415de2922ffafe 2024-11-19T17:50:33,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/2a22d96cd9ba407385415de2922ffafe, entries=150, sequenceid=387, filesize=12.0 K 2024-11-19T17:50:33,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 65f2046e0a9c76be2cfb08c4db762a3f in 471ms, sequenceid=387, compaction requested=false 2024-11-19T17:50:33,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 65f2046e0a9c76be2cfb08c4db762a3f: 2024-11-19T17:50:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:50:33,189 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1496 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4488 rows 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1499 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4496 rows 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1507 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4516 rows 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1509 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4526 rows 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1503 2024-11-19T17:50:33,189 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4508 rows 2024-11-19T17:50:33,189 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:50:33,190 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:50:33,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:33,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:33,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:33,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:50:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:50:33,195 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038633194"}]},"ts":"1732038633194"} 2024-11-19T17:50:33,197 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:50:33,198 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:50:33,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:50:33,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, UNASSIGN}] 2024-11-19T17:50:33,201 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, UNASSIGN 2024-11-19T17:50:33,203 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=65f2046e0a9c76be2cfb08c4db762a3f, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:33,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, UNASSIGN because future has completed 2024-11-19T17:50:33,205 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:50:33,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:50:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:50:33,358 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:33,358 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:50:33,358 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 65f2046e0a9c76be2cfb08c4db762a3f, disabling compactions & flushes 2024-11-19T17:50:33,358 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:33,358 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:33,358 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. after waiting 0 ms 2024-11-19T17:50:33,358 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:33,358 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 65f2046e0a9c76be2cfb08c4db762a3f 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T17:50:33,362 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/711e7541aba4456eac82034f1e629c90 is 50, key is test_row_0/A:col10/1732038632949/Put/seqid=0 2024-11-19T17:50:33,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742365_1541 (size=9857) 2024-11-19T17:50:33,373 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/711e7541aba4456eac82034f1e629c90 2024-11-19T17:50:33,388 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/ba1662bc59694d059bffd2c733a5318f is 50, key is test_row_0/B:col10/1732038632949/Put/seqid=0 2024-11-19T17:50:33,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742366_1542 (size=9857) 2024-11-19T17:50:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:50:33,793 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/ba1662bc59694d059bffd2c733a5318f 2024-11-19T17:50:33,798 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/056a663465dd42b3bc07524a837e921e is 50, key is test_row_0/C:col10/1732038632949/Put/seqid=0 2024-11-19T17:50:33,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742367_1543 (size=9857) 2024-11-19T17:50:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:50:34,202 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/056a663465dd42b3bc07524a837e921e 2024-11-19T17:50:34,206 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/A/711e7541aba4456eac82034f1e629c90 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711e7541aba4456eac82034f1e629c90 2024-11-19T17:50:34,209 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711e7541aba4456eac82034f1e629c90, entries=100, sequenceid=397, filesize=9.6 K 2024-11-19T17:50:34,210 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/B/ba1662bc59694d059bffd2c733a5318f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/ba1662bc59694d059bffd2c733a5318f 2024-11-19T17:50:34,213 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/ba1662bc59694d059bffd2c733a5318f, entries=100, sequenceid=397, filesize=9.6 K 2024-11-19T17:50:34,213 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/.tmp/C/056a663465dd42b3bc07524a837e921e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/056a663465dd42b3bc07524a837e921e 2024-11-19T17:50:34,216 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/056a663465dd42b3bc07524a837e921e, entries=100, sequenceid=397, filesize=9.6 K 2024-11-19T17:50:34,217 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 65f2046e0a9c76be2cfb08c4db762a3f in 859ms, sequenceid=397, compaction requested=true 2024-11-19T17:50:34,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/950afa748eb748d8a929e5bf0d8cec1b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef6d51b75ffb4c808a173380f73f179e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ae12e7bb363543ef899f925a589509b7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4d7a26744a3e423fabae9b722bea6151, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/dc6e8689e67e48a09c2788790671f803, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ab08836f0f9844d0acc853e9c8898aa8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ea67f771692644e59acbcf85bd0b7ebd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/38c9797e72e54beea565730e5fe22a7c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/b3fbdc5809dd4b7f82840047e0431d92, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/1299eb8f806f4ecdbd0d1c0cc936af62, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/30c84fa4f9ed42a7ac0415a555ad7723, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/124c73cff8624cae9b88ec1a0b0cadfb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711572d509334fae8d4f81a82d842f44, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/410145f46e4c4891954f102b1ceb30b0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef24974c9c414f2ebd8a1d57e0d7c9e4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ec51db0a35d14423b8085965311694f1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4757d12b37944edda491bfc59ac8d8cd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/82f6386047144e45b1971aa40c09c1c6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/7f044e6e9e464afcb2c416bd80f064eb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/8e6d0b2f30fa4cddb87a7c56d2babce0, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/3e54a90d8ad049238a6a6ba7b050fe89, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/fdaffc2b083a4ee092f54cdfd962968e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4caf3aeb8e104275a24478f8aaa53eca, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/500fc0cf675f4700873ec8d06c07b7ee, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/a9d823ff0fe24f3f96cda2e7002e7fae, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/321710c37c7a4c31b54ad9c5fef6aad1] to archive 2024-11-19T17:50:34,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:34,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/950afa748eb748d8a929e5bf0d8cec1b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/950afa748eb748d8a929e5bf0d8cec1b 2024-11-19T17:50:34,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef6d51b75ffb4c808a173380f73f179e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef6d51b75ffb4c808a173380f73f179e 2024-11-19T17:50:34,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ae12e7bb363543ef899f925a589509b7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ae12e7bb363543ef899f925a589509b7 2024-11-19T17:50:34,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4d7a26744a3e423fabae9b722bea6151 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4d7a26744a3e423fabae9b722bea6151 2024-11-19T17:50:34,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/dc6e8689e67e48a09c2788790671f803 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/dc6e8689e67e48a09c2788790671f803 2024-11-19T17:50:34,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ab08836f0f9844d0acc853e9c8898aa8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ab08836f0f9844d0acc853e9c8898aa8 2024-11-19T17:50:34,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ea67f771692644e59acbcf85bd0b7ebd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ea67f771692644e59acbcf85bd0b7ebd 2024-11-19T17:50:34,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/38c9797e72e54beea565730e5fe22a7c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/38c9797e72e54beea565730e5fe22a7c 2024-11-19T17:50:34,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/b3fbdc5809dd4b7f82840047e0431d92 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/b3fbdc5809dd4b7f82840047e0431d92 2024-11-19T17:50:34,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/1299eb8f806f4ecdbd0d1c0cc936af62 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/1299eb8f806f4ecdbd0d1c0cc936af62 2024-11-19T17:50:34,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/30c84fa4f9ed42a7ac0415a555ad7723 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/30c84fa4f9ed42a7ac0415a555ad7723 2024-11-19T17:50:34,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/124c73cff8624cae9b88ec1a0b0cadfb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/124c73cff8624cae9b88ec1a0b0cadfb 2024-11-19T17:50:34,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711572d509334fae8d4f81a82d842f44 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711572d509334fae8d4f81a82d842f44 2024-11-19T17:50:34,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/410145f46e4c4891954f102b1ceb30b0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/410145f46e4c4891954f102b1ceb30b0 2024-11-19T17:50:34,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef24974c9c414f2ebd8a1d57e0d7c9e4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ef24974c9c414f2ebd8a1d57e0d7c9e4 2024-11-19T17:50:34,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ec51db0a35d14423b8085965311694f1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/ec51db0a35d14423b8085965311694f1 2024-11-19T17:50:34,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4757d12b37944edda491bfc59ac8d8cd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4757d12b37944edda491bfc59ac8d8cd 2024-11-19T17:50:34,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/82f6386047144e45b1971aa40c09c1c6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/82f6386047144e45b1971aa40c09c1c6 2024-11-19T17:50:34,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/7f044e6e9e464afcb2c416bd80f064eb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/7f044e6e9e464afcb2c416bd80f064eb 2024-11-19T17:50:34,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/8e6d0b2f30fa4cddb87a7c56d2babce0 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/8e6d0b2f30fa4cddb87a7c56d2babce0 2024-11-19T17:50:34,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/3e54a90d8ad049238a6a6ba7b050fe89 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/3e54a90d8ad049238a6a6ba7b050fe89 2024-11-19T17:50:34,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/fdaffc2b083a4ee092f54cdfd962968e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/fdaffc2b083a4ee092f54cdfd962968e 2024-11-19T17:50:34,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4caf3aeb8e104275a24478f8aaa53eca to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/4caf3aeb8e104275a24478f8aaa53eca 2024-11-19T17:50:34,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/500fc0cf675f4700873ec8d06c07b7ee to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/500fc0cf675f4700873ec8d06c07b7ee 2024-11-19T17:50:34,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/a9d823ff0fe24f3f96cda2e7002e7fae to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/a9d823ff0fe24f3f96cda2e7002e7fae 2024-11-19T17:50:34,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/321710c37c7a4c31b54ad9c5fef6aad1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/321710c37c7a4c31b54ad9c5fef6aad1 2024-11-19T17:50:34,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/08b68c2e74a64d57b7939dc477aa20f1, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c0501ee838ab4027a099000a3d298993, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/baa5ab55db46463e956212f8988e9eef, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0c9b946ad40f4b268a5e488b6b383b3f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3690097a20b4459c9987675e98173695, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d3b2edb25e864720b638d36821b7a1bf, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/f19191260672431cb05e1116f9cbab07, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3ba8b101d94c40c09375516b716e750c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/b962ddf0ae8946f099bda6e6cb4ddd74, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/51388e89fc2c4eed940c2466c218d87a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5c718c7ad70a47c69700a3e76fcf6e64, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/111211513f0249028e2d924dc6094479, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d81c471e02ae49c4ba57f746e2a36e57, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/6844a7545a554259a042f21a3a58eaff, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5048d7214a694d4ab4c1e84d0da46f87, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/37c44a41f5154284927c22c7a35e2b37, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/1a958bada20f4bcebd956b876f249e2f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/95e396e6c14a4594b20b4c8ba7e2ca67, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5fe00cf1a03d418fbbb5ab06d7aa5105, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0d1d6b4ada3344e2bb77e4b69301ebe4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c6f59f4b940143b7bebd5845d4a1f117, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c314aaef66064f4dac9989f4cf750a71, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/64622981be3343349018018035904954, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3a28069861824619b64db1c742cb90b8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/376b87e3c56f43ff896cd3f6d369da6b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/146c59715bad47c6bed0cab037eb746f] to archive 2024-11-19T17:50:34,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:34,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/08b68c2e74a64d57b7939dc477aa20f1 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/08b68c2e74a64d57b7939dc477aa20f1 2024-11-19T17:50:34,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c0501ee838ab4027a099000a3d298993 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c0501ee838ab4027a099000a3d298993 2024-11-19T17:50:34,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/baa5ab55db46463e956212f8988e9eef to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/baa5ab55db46463e956212f8988e9eef 2024-11-19T17:50:34,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0c9b946ad40f4b268a5e488b6b383b3f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0c9b946ad40f4b268a5e488b6b383b3f 2024-11-19T17:50:34,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3690097a20b4459c9987675e98173695 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3690097a20b4459c9987675e98173695 2024-11-19T17:50:34,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d3b2edb25e864720b638d36821b7a1bf to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d3b2edb25e864720b638d36821b7a1bf 2024-11-19T17:50:34,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/f19191260672431cb05e1116f9cbab07 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/f19191260672431cb05e1116f9cbab07 2024-11-19T17:50:34,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3ba8b101d94c40c09375516b716e750c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3ba8b101d94c40c09375516b716e750c 2024-11-19T17:50:34,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/b962ddf0ae8946f099bda6e6cb4ddd74 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/b962ddf0ae8946f099bda6e6cb4ddd74 2024-11-19T17:50:34,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/51388e89fc2c4eed940c2466c218d87a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/51388e89fc2c4eed940c2466c218d87a 2024-11-19T17:50:34,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5c718c7ad70a47c69700a3e76fcf6e64 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5c718c7ad70a47c69700a3e76fcf6e64 2024-11-19T17:50:34,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/111211513f0249028e2d924dc6094479 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/111211513f0249028e2d924dc6094479 2024-11-19T17:50:34,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d81c471e02ae49c4ba57f746e2a36e57 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/d81c471e02ae49c4ba57f746e2a36e57 2024-11-19T17:50:34,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/6844a7545a554259a042f21a3a58eaff to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/6844a7545a554259a042f21a3a58eaff 2024-11-19T17:50:34,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5048d7214a694d4ab4c1e84d0da46f87 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5048d7214a694d4ab4c1e84d0da46f87 2024-11-19T17:50:34,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/37c44a41f5154284927c22c7a35e2b37 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/37c44a41f5154284927c22c7a35e2b37 2024-11-19T17:50:34,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/1a958bada20f4bcebd956b876f249e2f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/1a958bada20f4bcebd956b876f249e2f 2024-11-19T17:50:34,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/95e396e6c14a4594b20b4c8ba7e2ca67 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/95e396e6c14a4594b20b4c8ba7e2ca67 2024-11-19T17:50:34,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5fe00cf1a03d418fbbb5ab06d7aa5105 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/5fe00cf1a03d418fbbb5ab06d7aa5105 2024-11-19T17:50:34,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0d1d6b4ada3344e2bb77e4b69301ebe4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0d1d6b4ada3344e2bb77e4b69301ebe4 2024-11-19T17:50:34,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c6f59f4b940143b7bebd5845d4a1f117 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c6f59f4b940143b7bebd5845d4a1f117 2024-11-19T17:50:34,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c314aaef66064f4dac9989f4cf750a71 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/c314aaef66064f4dac9989f4cf750a71 2024-11-19T17:50:34,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/64622981be3343349018018035904954 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/64622981be3343349018018035904954 2024-11-19T17:50:34,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3a28069861824619b64db1c742cb90b8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/3a28069861824619b64db1c742cb90b8 2024-11-19T17:50:34,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/376b87e3c56f43ff896cd3f6d369da6b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/376b87e3c56f43ff896cd3f6d369da6b 2024-11-19T17:50:34,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/146c59715bad47c6bed0cab037eb746f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/146c59715bad47c6bed0cab037eb746f 2024-11-19T17:50:34,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/55acbdab0134482a929ac8c9ae0da508, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/597059c295664237980baf5899d89c7f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/7535ed8f75cd47e2952707b9ca2e8a38, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/27ba2a4f9f154e3b88dc749badb6884c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/6c3c2979f0ce41c98157467d926fdae5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d4683d10f9f54eaaa689eba6d4305805, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9d8a0560f1234f8f82ca449f71eae8e5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/320d857fffcf4b309b58c5931bae45a3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/4274b985fcd740a2b54d9b8ad7c16aac, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/24dced735dc74db7a414a382d83e1a92, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/f9b854e1513d41debf665482185e5ac2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/892f3c51a5ea4ab0b05700371cc004df, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/8c4241cb27af4c87a09ab2ef03348f45, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/93c6fd867d024058abc10cadbb6afbe6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fef18ae77c124d75a01d81c7c8a2e0b2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a64e6fc3d89c484aba4958dc25db7f16, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/b964c8723a234842813dc47d2fa8f0a8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/0cbded8c938d4c1786a5aa52e8c25b44, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/96b9f606021342d29223a148a9e4a6a5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a13a4613c110407489f1f84da5e38ecf, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/bd5b15837dba49e4b7cc551c609ecf99, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9f675c2224b94e7aa830a7b0a36e7fdb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/21d18cc7a5404b9c8a228336669a881d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/94d5f938313f41ce9c3b72c8769c1cfa, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/921fa90ece354452a6936fdc2c9dcbea, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fc39aaa77ddb42ae94bce2a11bd4299f] to archive 2024-11-19T17:50:34,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:50:34,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/55acbdab0134482a929ac8c9ae0da508 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/55acbdab0134482a929ac8c9ae0da508 2024-11-19T17:50:34,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/597059c295664237980baf5899d89c7f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/597059c295664237980baf5899d89c7f 2024-11-19T17:50:34,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/7535ed8f75cd47e2952707b9ca2e8a38 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/7535ed8f75cd47e2952707b9ca2e8a38 2024-11-19T17:50:34,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/27ba2a4f9f154e3b88dc749badb6884c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/27ba2a4f9f154e3b88dc749badb6884c 2024-11-19T17:50:34,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/6c3c2979f0ce41c98157467d926fdae5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/6c3c2979f0ce41c98157467d926fdae5 2024-11-19T17:50:34,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d4683d10f9f54eaaa689eba6d4305805 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d4683d10f9f54eaaa689eba6d4305805 2024-11-19T17:50:34,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9d8a0560f1234f8f82ca449f71eae8e5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9d8a0560f1234f8f82ca449f71eae8e5 2024-11-19T17:50:34,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/320d857fffcf4b309b58c5931bae45a3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/320d857fffcf4b309b58c5931bae45a3 2024-11-19T17:50:34,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/4274b985fcd740a2b54d9b8ad7c16aac to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/4274b985fcd740a2b54d9b8ad7c16aac 2024-11-19T17:50:34,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/24dced735dc74db7a414a382d83e1a92 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/24dced735dc74db7a414a382d83e1a92 2024-11-19T17:50:34,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/f9b854e1513d41debf665482185e5ac2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/f9b854e1513d41debf665482185e5ac2 2024-11-19T17:50:34,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/892f3c51a5ea4ab0b05700371cc004df to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/892f3c51a5ea4ab0b05700371cc004df 2024-11-19T17:50:34,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/8c4241cb27af4c87a09ab2ef03348f45 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/8c4241cb27af4c87a09ab2ef03348f45 2024-11-19T17:50:34,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/93c6fd867d024058abc10cadbb6afbe6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/93c6fd867d024058abc10cadbb6afbe6 2024-11-19T17:50:34,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fef18ae77c124d75a01d81c7c8a2e0b2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fef18ae77c124d75a01d81c7c8a2e0b2 2024-11-19T17:50:34,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a64e6fc3d89c484aba4958dc25db7f16 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a64e6fc3d89c484aba4958dc25db7f16 2024-11-19T17:50:34,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/b964c8723a234842813dc47d2fa8f0a8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/b964c8723a234842813dc47d2fa8f0a8 2024-11-19T17:50:34,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/0cbded8c938d4c1786a5aa52e8c25b44 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/0cbded8c938d4c1786a5aa52e8c25b44 2024-11-19T17:50:34,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/96b9f606021342d29223a148a9e4a6a5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/96b9f606021342d29223a148a9e4a6a5 2024-11-19T17:50:34,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a13a4613c110407489f1f84da5e38ecf to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/a13a4613c110407489f1f84da5e38ecf 2024-11-19T17:50:34,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/bd5b15837dba49e4b7cc551c609ecf99 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/bd5b15837dba49e4b7cc551c609ecf99 2024-11-19T17:50:34,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9f675c2224b94e7aa830a7b0a36e7fdb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/9f675c2224b94e7aa830a7b0a36e7fdb 2024-11-19T17:50:34,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/21d18cc7a5404b9c8a228336669a881d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/21d18cc7a5404b9c8a228336669a881d 2024-11-19T17:50:34,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/94d5f938313f41ce9c3b72c8769c1cfa to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/94d5f938313f41ce9c3b72c8769c1cfa 2024-11-19T17:50:34,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/921fa90ece354452a6936fdc2c9dcbea to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/921fa90ece354452a6936fdc2c9dcbea 2024-11-19T17:50:34,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fc39aaa77ddb42ae94bce2a11bd4299f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/fc39aaa77ddb42ae94bce2a11bd4299f 2024-11-19T17:50:34,307 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/recovered.edits/400.seqid, newMaxSeqId=400, maxSeqId=1 2024-11-19T17:50:34,308 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. 2024-11-19T17:50:34,308 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 65f2046e0a9c76be2cfb08c4db762a3f: Waiting for close lock at 1732038633358Running coprocessor pre-close hooks at 1732038633358Disabling compacts and flushes for region at 1732038633358Disabling writes for close at 1732038633358Obtaining lock to block concurrent updates at 1732038633358Preparing flush snapshotting stores in 65f2046e0a9c76be2cfb08c4db762a3f at 1732038633358Finished memstore snapshotting TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732038633359 (+1 ms)Flushing stores of TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f. at 1732038633359Flushing 65f2046e0a9c76be2cfb08c4db762a3f/A: creating writer at 1732038633359Flushing 65f2046e0a9c76be2cfb08c4db762a3f/A: appending metadata at 1732038633362 (+3 ms)Flushing 65f2046e0a9c76be2cfb08c4db762a3f/A: closing flushed file at 1732038633362Flushing 65f2046e0a9c76be2cfb08c4db762a3f/B: creating writer at 1732038633378 (+16 ms)Flushing 65f2046e0a9c76be2cfb08c4db762a3f/B: appending metadata at 1732038633387 (+9 ms)Flushing 65f2046e0a9c76be2cfb08c4db762a3f/B: closing flushed file at 1732038633387Flushing 65f2046e0a9c76be2cfb08c4db762a3f/C: creating writer at 1732038633796 (+409 ms)Flushing 65f2046e0a9c76be2cfb08c4db762a3f/C: appending metadata at 1732038633798 (+2 ms)Flushing 65f2046e0a9c76be2cfb08c4db762a3f/C: closing flushed file at 1732038633798Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@364d5332: reopening flushed file at 1732038634205 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@367b6eef: reopening flushed file at 1732038634209 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6005fdb0: reopening flushed file at 1732038634213 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 65f2046e0a9c76be2cfb08c4db762a3f in 859ms, sequenceid=397, compaction requested=true at 1732038634217 (+4 ms)Writing region close event to WAL at 1732038634304 (+87 ms)Running coprocessor post-close hooks at 1732038634308 (+4 ms)Closed at 1732038634308 2024-11-19T17:50:34,310 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:34,311 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=65f2046e0a9c76be2cfb08c4db762a3f, regionState=CLOSED 2024-11-19T17:50:34,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:50:34,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T17:50:34,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 65f2046e0a9c76be2cfb08c4db762a3f, server=26d3106abcfe,36839,1732038481522 in 1.1080 sec 2024-11-19T17:50:34,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-19T17:50:34,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=65f2046e0a9c76be2cfb08c4db762a3f, UNASSIGN in 1.1140 sec 2024-11-19T17:50:34,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T17:50:34,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1180 sec 2024-11-19T17:50:34,319 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038634319"}]},"ts":"1732038634319"} 2024-11-19T17:50:34,321 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:50:34,321 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:50:34,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1310 sec 2024-11-19T17:50:34,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:50:34,329 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:34,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:50:34,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:34,331 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:34,332 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T17:50:34,333 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:34,335 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/recovered.edits] 2024-11-19T17:50:34,337 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/52020425ebb548499eb28c84ec4dd683 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/52020425ebb548499eb28c84ec4dd683 2024-11-19T17:50:34,338 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711e7541aba4456eac82034f1e629c90 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/711e7541aba4456eac82034f1e629c90 2024-11-19T17:50:34,339 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/c60a130a9d9c46f2b935236380a69372 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/A/c60a130a9d9c46f2b935236380a69372 2024-11-19T17:50:34,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0efab8e2fca44881ac84682a9e4dcf0f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/0efab8e2fca44881ac84682a9e4dcf0f 2024-11-19T17:50:34,343 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/14fcef1603ba4ac6ae15c8d56e1314ba to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/14fcef1603ba4ac6ae15c8d56e1314ba 2024-11-19T17:50:34,344 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/ba1662bc59694d059bffd2c733a5318f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/B/ba1662bc59694d059bffd2c733a5318f 2024-11-19T17:50:34,346 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/056a663465dd42b3bc07524a837e921e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/056a663465dd42b3bc07524a837e921e 2024-11-19T17:50:34,347 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/2a22d96cd9ba407385415de2922ffafe to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/2a22d96cd9ba407385415de2922ffafe 2024-11-19T17:50:34,349 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d63ed31d48db449d97034b8038c24d40 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/C/d63ed31d48db449d97034b8038c24d40 2024-11-19T17:50:34,352 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/recovered.edits/400.seqid to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f/recovered.edits/400.seqid 2024-11-19T17:50:34,352 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/65f2046e0a9c76be2cfb08c4db762a3f 2024-11-19T17:50:34,352 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:50:34,355 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:34,357 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:50:34,359 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:50:34,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:34,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:50:34,361 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732038634360"}]},"ts":"9223372036854775807"} 2024-11-19T17:50:34,363 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:50:34,363 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 65f2046e0a9c76be2cfb08c4db762a3f, NAME => 'TestAcidGuarantees,,1732038609725.65f2046e0a9c76be2cfb08c4db762a3f.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:50:34,363 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:50:34,363 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732038634363"}]},"ts":"9223372036854775807"} 2024-11-19T17:50:34,366 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:50:34,367 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:34,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-19T17:50:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T17:50:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:50:34,441 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:34,447 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 240), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=617 (was 644), ProcessCount=11 (was 11), AvailableMemoryMB=10075 (was 9867) - AvailableMemoryMB LEAK? - 2024-11-19T17:50:34,456 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=617, ProcessCount=12, AvailableMemoryMB=10075 2024-11-19T17:50:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:50:34,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:50:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:34,461 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:50:34,461 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:34,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-19T17:50:34,463 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:50:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T17:50:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742368_1544 (size=959) 2024-11-19T17:50:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T17:50:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T17:50:34,869 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d 2024-11-19T17:50:34,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742369_1545 (size=53) 2024-11-19T17:50:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T17:50:35,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:50:35,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b9c37934b4aa3330f167e8a793f13680, disabling compactions & flushes 2024-11-19T17:50:35,276 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:35,276 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:35,276 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. after waiting 0 ms 2024-11-19T17:50:35,276 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:35,276 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:35,276 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b9c37934b4aa3330f167e8a793f13680: Waiting for close lock at 1732038635275Disabling compacts and flushes for region at 1732038635275Disabling writes for close at 1732038635276 (+1 ms)Writing region close event to WAL at 1732038635276Closed at 1732038635276 2024-11-19T17:50:35,277 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:50:35,277 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732038635277"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732038635277"}]},"ts":"1732038635277"} 2024-11-19T17:50:35,278 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:50:35,279 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:50:35,279 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038635279"}]},"ts":"1732038635279"} 2024-11-19T17:50:35,281 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:50:35,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, ASSIGN}] 2024-11-19T17:50:35,282 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, ASSIGN 2024-11-19T17:50:35,283 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, ASSIGN; state=OFFLINE, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=false 2024-11-19T17:50:35,433 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=b9c37934b4aa3330f167e8a793f13680, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:35,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, ASSIGN because future has completed 2024-11-19T17:50:35,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:50:35,589 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:35,589 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:50:35,589 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,589 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:50:35,589 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,589 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,590 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,591 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9c37934b4aa3330f167e8a793f13680 columnFamilyName A 2024-11-19T17:50:35,591 DEBUG [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:35,592 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(327): Store=b9c37934b4aa3330f167e8a793f13680/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:35,592 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,592 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9c37934b4aa3330f167e8a793f13680 columnFamilyName B 2024-11-19T17:50:35,592 DEBUG [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:35,593 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(327): Store=b9c37934b4aa3330f167e8a793f13680/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:35,593 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,594 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9c37934b4aa3330f167e8a793f13680 columnFamilyName C 2024-11-19T17:50:35,594 DEBUG [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:35,594 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(327): Store=b9c37934b4aa3330f167e8a793f13680/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:35,594 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,594 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,595 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,595 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,595 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,596 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:50:35,596 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,598 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:50:35,598 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened b9c37934b4aa3330f167e8a793f13680; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64212722, jitterRate=-0.04315587878227234}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:50:35,598 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:35,599 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for b9c37934b4aa3330f167e8a793f13680: Running coprocessor pre-open hook at 1732038635590Writing region info on filesystem at 1732038635590Initializing all the Stores at 1732038635590Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038635590Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038635590Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038635590Cleaning up temporary data from old regions at 1732038635595 (+5 ms)Running coprocessor post-open hooks at 1732038635598 (+3 ms)Region opened successfully at 1732038635599 (+1 ms) 2024-11-19T17:50:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T17:50:35,599 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., pid=148, masterSystemTime=1732038635587 2024-11-19T17:50:35,601 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:35,601 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:35,601 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=b9c37934b4aa3330f167e8a793f13680, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:35,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:50:35,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-19T17:50:35,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 in 168 msec 2024-11-19T17:50:35,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T17:50:35,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, ASSIGN in 329 msec 2024-11-19T17:50:35,613 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:50:35,613 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038635613"}]},"ts":"1732038635613"} 2024-11-19T17:50:35,614 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:50:35,615 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:50:35,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-11-19T17:50:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T17:50:36,609 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:36,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:36,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:36,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:36,611 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9f803e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:36,613 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:36,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce937f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:36,614 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:36,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:36,617 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:50:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,41481,1732038480627 2024-11-19T17:50:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64aa018d 2024-11-19T17:50:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:50:36,618 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:50:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:50:36,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:50:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T17:50:36,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742370_1546 (size=995) 2024-11-19T17:50:37,042 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T17:50:37,042 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T17:50:37,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:50:37,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, REOPEN/MOVE}] 2024-11-19T17:50:37,047 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, REOPEN/MOVE 2024-11-19T17:50:37,048 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=b9c37934b4aa3330f167e8a793f13680, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, REOPEN/MOVE because future has completed 2024-11-19T17:50:37,049 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:50:37,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:50:37,202 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,202 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:50:37,202 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing b9c37934b4aa3330f167e8a793f13680, disabling compactions & flushes 2024-11-19T17:50:37,202 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,202 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,202 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. after waiting 0 ms 2024-11-19T17:50:37,202 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,204 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T17:50:37,205 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,205 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for b9c37934b4aa3330f167e8a793f13680: Waiting for close lock at 1732038637202Running coprocessor pre-close hooks at 1732038637202Disabling compacts and flushes for region at 1732038637202Disabling writes for close at 1732038637202Writing region close event to WAL at 1732038637202Running coprocessor post-close hooks at 1732038637205 (+3 ms)Closed at 1732038637205 2024-11-19T17:50:37,205 WARN [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: b9c37934b4aa3330f167e8a793f13680 to self. 2024-11-19T17:50:37,207 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,207 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=b9c37934b4aa3330f167e8a793f13680, regionState=CLOSED 2024-11-19T17:50:37,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:50:37,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-19T17:50:37,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 in 160 msec 2024-11-19T17:50:37,211 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, REOPEN/MOVE; state=CLOSED, location=26d3106abcfe,36839,1732038481522; forceNewPlan=false, retain=true 2024-11-19T17:50:37,361 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=b9c37934b4aa3330f167e8a793f13680, regionState=OPENING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, REOPEN/MOVE because future has completed 2024-11-19T17:50:37,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:50:37,517 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,517 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:50:37,517 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,517 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:50:37,518 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,518 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,519 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,520 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9c37934b4aa3330f167e8a793f13680 columnFamilyName A 2024-11-19T17:50:37,520 DEBUG [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:37,521 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(327): Store=b9c37934b4aa3330f167e8a793f13680/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:37,521 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,522 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9c37934b4aa3330f167e8a793f13680 columnFamilyName B 2024-11-19T17:50:37,522 DEBUG [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:37,522 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(327): Store=b9c37934b4aa3330f167e8a793f13680/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:37,522 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,523 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9c37934b4aa3330f167e8a793f13680 columnFamilyName C 2024-11-19T17:50:37,523 DEBUG [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:37,523 INFO [StoreOpener-b9c37934b4aa3330f167e8a793f13680-1 {}] regionserver.HStore(327): Store=b9c37934b4aa3330f167e8a793f13680/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:37,524 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,524 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,525 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,525 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,525 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,526 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:50:37,527 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,527 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened b9c37934b4aa3330f167e8a793f13680; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70126237, jitterRate=0.044962361454963684}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:50:37,527 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,528 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for b9c37934b4aa3330f167e8a793f13680: Running coprocessor pre-open hook at 1732038637518Writing region info on filesystem at 1732038637518Initializing all the Stores at 1732038637518Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038637518Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038637519 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038637519Cleaning up temporary data from old regions at 1732038637525 (+6 ms)Running coprocessor post-open hooks at 1732038637527 (+2 ms)Region opened successfully at 1732038637528 (+1 ms) 2024-11-19T17:50:37,528 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., pid=153, masterSystemTime=1732038637515 2024-11-19T17:50:37,530 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,530 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,530 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=b9c37934b4aa3330f167e8a793f13680, regionState=OPEN, openSeqNum=5, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:50:37,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-19T17:50:37,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 in 169 msec 2024-11-19T17:50:37,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T17:50:37,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, REOPEN/MOVE in 487 msec 2024-11-19T17:50:37,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-19T17:50:37,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-19T17:50:37,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 918 msec 2024-11-19T17:50:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-19T17:50:37,540 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:37,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,541 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e5d55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,543 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dbbc069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,545 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc6a711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,546 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655aa853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,548 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@372d0674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,550 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6431186, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,552 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47df27c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,553 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e98b07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,555 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff4c29e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,556 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f221450, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,558 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@834eb8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,559 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b425591, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,561 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4745b9a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,562 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1b792c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,565 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744e65d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,566 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3596bb00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,568 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67895da9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21aceedb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,41481,-1 for getting cluster id 2024-11-19T17:50:37,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:50:37,571 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '27879909-b964-46e7-ae0b-b7fd34d9080a' 2024-11-19T17:50:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:50:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "27879909-b964-46e7-ae0b-b7fd34d9080a" 2024-11-19T17:50:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d87ec31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,41481,-1] 2024-11-19T17:50:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:50:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:37,572 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:50:37,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d40d308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:50:37,573 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,573 DEBUG [Thread-2448 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,574 DEBUG [Thread-2446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:37,574 DEBUG [Thread-2435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,575 DEBUG [Thread-2440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T17:50:37,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,575 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,575 DEBUG [Thread-2442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,576 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T17:50:37,576 DEBUG [Thread-2429 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,576 DEBUG [Thread-2444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,576 DEBUG [Thread-2429 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,577 DEBUG [Thread-2437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,578 DEBUG [Thread-2433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:50:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,580 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:37,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,581 DEBUG [Thread-2429 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,583 DEBUG [Thread-2433 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,36839,1732038481522, seqNum=-1] 2024-11-19T17:50:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,587 DEBUG [Thread-2433 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:50:37,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:50:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5] 2024-11-19T17:50:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:37,594 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:37,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T17:50:37,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038697609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038697610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038697614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b392cdae497a4c2ea30a79cdd727f526_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038637593/Put/seqid=0 2024-11-19T17:50:37,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038697618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742371_1547 (size=12154) 2024-11-19T17:50:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038697625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T17:50:37,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038697728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038697728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038697728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038697728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038697728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:37,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:37,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T17:50:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:37,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038697938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038697938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038697938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038697938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038697938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:38,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b392cdae497a4c2ea30a79cdd727f526_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b392cdae497a4c2ea30a79cdd727f526_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:38,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/231407d5fef34092afb6d4cdefaf2350, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:38,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/231407d5fef34092afb6d4cdefaf2350 is 175, key is test_row_0/A:col10/1732038637593/Put/seqid=0 2024-11-19T17:50:38,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742372_1548 (size=30955) 2024-11-19T17:50:38,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:38,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:38,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T17:50:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038698249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038698249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038698249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038698249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038698260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:38,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:38,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,446 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/231407d5fef34092afb6d4cdefaf2350 2024-11-19T17:50:38,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/24c81d66f175452fa8da48a51b9d3552 is 50, key is test_row_0/B:col10/1732038637593/Put/seqid=0 2024-11-19T17:50:38,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742373_1549 (size=12001) 2024-11-19T17:50:38,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:38,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:38,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:38,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:38,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T17:50:38,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038698759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038698759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038698759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038698759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038698768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:38,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/24c81d66f175452fa8da48a51b9d3552 2024-11-19T17:50:38,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/0a0d706cbbff4338822c91c308baabeb is 50, key is test_row_0/C:col10/1732038637593/Put/seqid=0 2024-11-19T17:50:38,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742374_1550 (size=12001) 2024-11-19T17:50:38,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:38,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:38,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:38,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:39,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:39,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:39,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:39,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:39,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:39,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:39,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:39,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/0a0d706cbbff4338822c91c308baabeb 2024-11-19T17:50:39,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/231407d5fef34092afb6d4cdefaf2350 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/231407d5fef34092afb6d4cdefaf2350 2024-11-19T17:50:39,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/231407d5fef34092afb6d4cdefaf2350, entries=150, sequenceid=18, filesize=30.2 K 2024-11-19T17:50:39,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/24c81d66f175452fa8da48a51b9d3552 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/24c81d66f175452fa8da48a51b9d3552 2024-11-19T17:50:39,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/24c81d66f175452fa8da48a51b9d3552, entries=150, sequenceid=18, filesize=11.7 K 2024-11-19T17:50:39,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/0a0d706cbbff4338822c91c308baabeb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0a0d706cbbff4338822c91c308baabeb 2024-11-19T17:50:39,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0a0d706cbbff4338822c91c308baabeb, entries=150, sequenceid=18, filesize=11.7 K 2024-11-19T17:50:39,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for b9c37934b4aa3330f167e8a793f13680 in 1751ms, sequenceid=18, compaction requested=false 2024-11-19T17:50:39,346 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:50:39,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:39,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T17:50:39,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:39,434 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T17:50:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111918cf2c06c0e2447890fc8f5000706043_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038637614/Put/seqid=0 2024-11-19T17:50:39,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742375_1551 (size=12154) 2024-11-19T17:50:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T17:50:39,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:39,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038699779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038699779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038699781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:39,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038699782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:39,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038699789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,793 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:39,827 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:50:39,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:39,878 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111918cf2c06c0e2447890fc8f5000706043_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111918cf2c06c0e2447890fc8f5000706043_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/ce007b4810f84836aa41832a7085930a, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/ce007b4810f84836aa41832a7085930a is 175, key is test_row_0/A:col10/1732038637614/Put/seqid=0 2024-11-19T17:50:39,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742376_1552 (size=30955) 2024-11-19T17:50:39,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038699889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:39,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038699889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:39,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038699889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:39,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038699889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038700098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038700099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038700101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038700101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,283 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/ce007b4810f84836aa41832a7085930a 2024-11-19T17:50:40,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/8e6a63d238ed49b893a4704a0b8b2d3d is 50, key is test_row_0/B:col10/1732038637614/Put/seqid=0 2024-11-19T17:50:40,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742377_1553 (size=12001) 2024-11-19T17:50:40,312 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/8e6a63d238ed49b893a4704a0b8b2d3d 2024-11-19T17:50:40,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/2cafb924a086462ab6bda3062f5708f4 is 50, key is test_row_0/C:col10/1732038637614/Put/seqid=0 2024-11-19T17:50:40,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742378_1554 (size=12001) 2024-11-19T17:50:40,331 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/2cafb924a086462ab6bda3062f5708f4 2024-11-19T17:50:40,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/ce007b4810f84836aa41832a7085930a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce007b4810f84836aa41832a7085930a 2024-11-19T17:50:40,342 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce007b4810f84836aa41832a7085930a, entries=150, sequenceid=41, filesize=30.2 K 2024-11-19T17:50:40,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/8e6a63d238ed49b893a4704a0b8b2d3d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/8e6a63d238ed49b893a4704a0b8b2d3d 2024-11-19T17:50:40,347 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/8e6a63d238ed49b893a4704a0b8b2d3d, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T17:50:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/2cafb924a086462ab6bda3062f5708f4 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2cafb924a086462ab6bda3062f5708f4 2024-11-19T17:50:40,351 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2cafb924a086462ab6bda3062f5708f4, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T17:50:40,353 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b9c37934b4aa3330f167e8a793f13680 in 919ms, sequenceid=41, compaction requested=false 2024-11-19T17:50:40,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:40,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:40,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T17:50:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T17:50:40,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T17:50:40,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7740 sec 2024-11-19T17:50:40,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.7830 sec 2024-11-19T17:50:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:40,411 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:40,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T17:50:40,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198e31ae259b8144428081c17c8cfc11a7_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038639777/Put/seqid=0 2024-11-19T17:50:40,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742379_1555 (size=14594) 2024-11-19T17:50:40,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038700438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038700438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038700439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038700443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038700549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038700549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038700549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038700558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038700759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038700759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038700759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038700768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:40,823 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:40,826 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198e31ae259b8144428081c17c8cfc11a7_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198e31ae259b8144428081c17c8cfc11a7_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:40,827 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b214be388bd64b00b06e6e74f300cd6f, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:40,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b214be388bd64b00b06e6e74f300cd6f is 175, key is test_row_0/A:col10/1732038639777/Put/seqid=0 2024-11-19T17:50:40,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742380_1556 (size=39549) 2024-11-19T17:50:41,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038701068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038701068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038701068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038701078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,232 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b214be388bd64b00b06e6e74f300cd6f 2024-11-19T17:50:41,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/706415366666419c98145e38e7e65b2a is 50, key is test_row_0/B:col10/1732038639777/Put/seqid=0 2024-11-19T17:50:41,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742381_1557 (size=12001) 2024-11-19T17:50:41,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/706415366666419c98145e38e7e65b2a 2024-11-19T17:50:41,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/26afb3745abf4a4093e88ab8adac3f02 is 50, key is test_row_0/C:col10/1732038639777/Put/seqid=0 2024-11-19T17:50:41,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742382_1558 (size=12001) 2024-11-19T17:50:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038701578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038701578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038701578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038701589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/26afb3745abf4a4093e88ab8adac3f02 2024-11-19T17:50:41,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b214be388bd64b00b06e6e74f300cd6f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b214be388bd64b00b06e6e74f300cd6f 2024-11-19T17:50:41,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b214be388bd64b00b06e6e74f300cd6f, entries=200, sequenceid=55, filesize=38.6 K 2024-11-19T17:50:41,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/706415366666419c98145e38e7e65b2a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/706415366666419c98145e38e7e65b2a 2024-11-19T17:50:41,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/706415366666419c98145e38e7e65b2a, entries=150, sequenceid=55, filesize=11.7 K 2024-11-19T17:50:41,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/26afb3745abf4a4093e88ab8adac3f02 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/26afb3745abf4a4093e88ab8adac3f02 2024-11-19T17:50:41,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/26afb3745abf4a4093e88ab8adac3f02, entries=150, sequenceid=55, filesize=11.7 K 2024-11-19T17:50:41,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b9c37934b4aa3330f167e8a793f13680 in 1271ms, sequenceid=55, compaction requested=true 2024-11-19T17:50:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:41,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:41,682 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:41,683 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:41,683 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:50:41,683 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:41,683 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/231407d5fef34092afb6d4cdefaf2350, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce007b4810f84836aa41832a7085930a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b214be388bd64b00b06e6e74f300cd6f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=99.1 K 2024-11-19T17:50:41,683 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:41,683 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/231407d5fef34092afb6d4cdefaf2350, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce007b4810f84836aa41832a7085930a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b214be388bd64b00b06e6e74f300cd6f] 2024-11-19T17:50:41,684 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:41,684 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:50:41,684 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:41,684 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/24c81d66f175452fa8da48a51b9d3552, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/8e6a63d238ed49b893a4704a0b8b2d3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/706415366666419c98145e38e7e65b2a] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=35.2 K 2024-11-19T17:50:41,686 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 231407d5fef34092afb6d4cdefaf2350, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732038637592 2024-11-19T17:50:41,686 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 24c81d66f175452fa8da48a51b9d3552, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732038637592 2024-11-19T17:50:41,686 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6a63d238ed49b893a4704a0b8b2d3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732038637603 2024-11-19T17:50:41,686 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce007b4810f84836aa41832a7085930a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732038637603 2024-11-19T17:50:41,687 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 706415366666419c98145e38e7e65b2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732038639777 2024-11-19T17:50:41,687 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b214be388bd64b00b06e6e74f300cd6f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732038639777 2024-11-19T17:50:41,698 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:41,698 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/80781c5431484c679d84c5ec9aafb33f is 50, key is test_row_0/B:col10/1732038639777/Put/seqid=0 2024-11-19T17:50:41,705 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:41,723 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119835396073577439cb03127c4b2a67ecb_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:41,725 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119835396073577439cb03127c4b2a67ecb_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:41,725 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119835396073577439cb03127c4b2a67ecb_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:41,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742383_1559 (size=12104) 2024-11-19T17:50:41,731 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/80781c5431484c679d84c5ec9aafb33f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/80781c5431484c679d84c5ec9aafb33f 2024-11-19T17:50:41,735 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into 80781c5431484c679d84c5ec9aafb33f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:41,735 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:41,735 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=13, startTime=1732038641682; duration=0sec 2024-11-19T17:50:41,735 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:41,735 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:50:41,735 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:41,737 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:41,737 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:50:41,737 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:41,737 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0a0d706cbbff4338822c91c308baabeb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2cafb924a086462ab6bda3062f5708f4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/26afb3745abf4a4093e88ab8adac3f02] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=35.2 K 2024-11-19T17:50:41,738 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a0d706cbbff4338822c91c308baabeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732038637592 2024-11-19T17:50:41,738 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cafb924a086462ab6bda3062f5708f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732038637603 2024-11-19T17:50:41,738 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 26afb3745abf4a4093e88ab8adac3f02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732038639777 2024-11-19T17:50:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T17:50:41,739 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:41,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T17:50:41,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:41,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:41,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T17:50:41,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742384_1560 (size=4469) 2024-11-19T17:50:41,758 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:41,759 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/7b5dc31aa7cf48b7930a6ce0b220aee3 is 50, key is test_row_0/C:col10/1732038639777/Put/seqid=0 2024-11-19T17:50:41,759 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#478 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:41,759 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/7c82b62b35c546cd83f2bbdc5b88b48b is 175, key is test_row_0/A:col10/1732038639777/Put/seqid=0 2024-11-19T17:50:41,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742385_1561 (size=12104) 2024-11-19T17:50:41,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742386_1562 (size=31058) 2024-11-19T17:50:41,797 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/7c82b62b35c546cd83f2bbdc5b88b48b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/7c82b62b35c546cd83f2bbdc5b88b48b 2024-11-19T17:50:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:41,801 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:41,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T17:50:41,804 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into 7c82b62b35c546cd83f2bbdc5b88b48b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:41,804 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:41,804 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=13, startTime=1732038641682; duration=0sec 2024-11-19T17:50:41,804 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:41,804 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:50:41,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b3aaaf1a078946709a54d27bf704d6fb_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038640441/Put/seqid=0 2024-11-19T17:50:41,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742387_1563 (size=14594) 2024-11-19T17:50:41,830 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:41,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b3aaaf1a078946709a54d27bf704d6fb_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b3aaaf1a078946709a54d27bf704d6fb_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:41,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/dfd1e7f607d04abd9f8a74c12c7938ef, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:41,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/dfd1e7f607d04abd9f8a74c12c7938ef is 175, key is test_row_0/A:col10/1732038640441/Put/seqid=0 2024-11-19T17:50:41,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742388_1564 (size=39549) 2024-11-19T17:50:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T17:50:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038701847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:41,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:41,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:41,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:41,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:41,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038701958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:42,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T17:50:42,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038702168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:42,196 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/7b5dc31aa7cf48b7930a6ce0b220aee3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/7b5dc31aa7cf48b7930a6ce0b220aee3 2024-11-19T17:50:42,200 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into 7b5dc31aa7cf48b7930a6ce0b220aee3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:42,200 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:42,200 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=13, startTime=1732038641682; duration=0sec 2024-11-19T17:50:42,200 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:42,200 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:50:42,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,237 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/dfd1e7f607d04abd9f8a74c12c7938ef 2024-11-19T17:50:42,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/39b8818c16f44206bd6edcb359772735 is 50, key is test_row_0/B:col10/1732038640441/Put/seqid=0 2024-11-19T17:50:42,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742389_1565 (size=12001) 2024-11-19T17:50:42,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T17:50:42,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038702479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:42,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:42,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:42,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038702588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:42,591 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:42,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038702598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:42,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038702598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038702598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:42,601 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,601 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:42,601 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:42,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/39b8818c16f44206bd6edcb359772735 2024-11-19T17:50:42,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/f578cc44c69e442ca7077102d5ff1d54 is 50, key is test_row_0/C:col10/1732038640441/Put/seqid=0 2024-11-19T17:50:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742390_1566 (size=12001) 2024-11-19T17:50:42,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T17:50:42,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:42,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038702988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:43,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/f578cc44c69e442ca7077102d5ff1d54 2024-11-19T17:50:43,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/dfd1e7f607d04abd9f8a74c12c7938ef as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/dfd1e7f607d04abd9f8a74c12c7938ef 2024-11-19T17:50:43,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/dfd1e7f607d04abd9f8a74c12c7938ef, entries=200, sequenceid=79, filesize=38.6 K 2024-11-19T17:50:43,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/39b8818c16f44206bd6edcb359772735 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/39b8818c16f44206bd6edcb359772735 2024-11-19T17:50:43,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/39b8818c16f44206bd6edcb359772735, entries=150, sequenceid=79, filesize=11.7 K 2024-11-19T17:50:43,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/f578cc44c69e442ca7077102d5ff1d54 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f578cc44c69e442ca7077102d5ff1d54 2024-11-19T17:50:43,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f578cc44c69e442ca7077102d5ff1d54, entries=150, sequenceid=79, filesize=11.7 K 2024-11-19T17:50:43,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b9c37934b4aa3330f167e8a793f13680 in 1317ms, sequenceid=79, compaction requested=false 2024-11-19T17:50:43,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T17:50:43,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:43,118 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:50:43,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119816e6a935fdc4b2b958b404afe2e6803_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038641822/Put/seqid=0 2024-11-19T17:50:43,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742391_1567 (size=12154) 2024-11-19T17:50:43,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:43,548 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119816e6a935fdc4b2b958b404afe2e6803_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119816e6a935fdc4b2b958b404afe2e6803_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/196f32edaa00460b9ecd1a3d4e9541c8, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/196f32edaa00460b9ecd1a3d4e9541c8 is 175, key is test_row_0/A:col10/1732038641822/Put/seqid=0 2024-11-19T17:50:43,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742392_1568 (size=30955) 2024-11-19T17:50:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T17:50:43,966 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/196f32edaa00460b9ecd1a3d4e9541c8 2024-11-19T17:50:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/1e4f9f851e72451da7e9598561f9e555 is 50, key is test_row_0/B:col10/1732038641822/Put/seqid=0 2024-11-19T17:50:43,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742393_1569 (size=12001) 2024-11-19T17:50:43,995 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/1e4f9f851e72451da7e9598561f9e555 2024-11-19T17:50:44,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/94fd63105abf4f17b4f228065bad3a4a is 50, key is test_row_0/C:col10/1732038641822/Put/seqid=0 2024-11-19T17:50:44,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742394_1570 (size=12001) 2024-11-19T17:50:44,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038704086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038704199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,409 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/94fd63105abf4f17b4f228065bad3a4a 2024-11-19T17:50:44,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038704409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/196f32edaa00460b9ecd1a3d4e9541c8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/196f32edaa00460b9ecd1a3d4e9541c8 2024-11-19T17:50:44,418 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/196f32edaa00460b9ecd1a3d4e9541c8, entries=150, sequenceid=94, filesize=30.2 K 2024-11-19T17:50:44,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/1e4f9f851e72451da7e9598561f9e555 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1e4f9f851e72451da7e9598561f9e555 2024-11-19T17:50:44,422 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1e4f9f851e72451da7e9598561f9e555, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:50:44,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/94fd63105abf4f17b4f228065bad3a4a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/94fd63105abf4f17b4f228065bad3a4a 2024-11-19T17:50:44,426 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/94fd63105abf4f17b4f228065bad3a4a, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:50:44,428 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b9c37934b4aa3330f167e8a793f13680 in 1309ms, sequenceid=94, compaction requested=true 2024-11-19T17:50:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:44,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T17:50:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T17:50:44,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T17:50:44,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6880 sec 2024-11-19T17:50:44,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.6930 sec 2024-11-19T17:50:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:44,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:44,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:50:44,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191bf07175ae5d40adb3923c71a4242620_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038644074/Put/seqid=0 2024-11-19T17:50:44,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038704625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038704627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038704629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038704635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742395_1571 (size=14594) 2024-11-19T17:50:44,646 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:44,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191bf07175ae5d40adb3923c71a4242620_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191bf07175ae5d40adb3923c71a4242620_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:44,650 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/3ab3f747b598411aa0e3f2385e3664c5, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:44,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/3ab3f747b598411aa0e3f2385e3664c5 is 175, key is test_row_0/A:col10/1732038644074/Put/seqid=0 2024-11-19T17:50:44,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742396_1572 (size=39549) 2024-11-19T17:50:44,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038704719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038704739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038704739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038704739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038704748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038704948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038704948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038704949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038704958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,053 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/3ab3f747b598411aa0e3f2385e3664c5 2024-11-19T17:50:45,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/fb2a7876d296427eb8f6e99ef00d9de5 is 50, key is test_row_0/B:col10/1732038644074/Put/seqid=0 2024-11-19T17:50:45,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742397_1573 (size=12001) 2024-11-19T17:50:45,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/fb2a7876d296427eb8f6e99ef00d9de5 2024-11-19T17:50:45,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/98df6ef2d25740b6aa9637fd9744b2d3 is 50, key is test_row_0/C:col10/1732038644074/Put/seqid=0 2024-11-19T17:50:45,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742398_1574 (size=12001) 2024-11-19T17:50:45,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/98df6ef2d25740b6aa9637fd9744b2d3 2024-11-19T17:50:45,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/3ab3f747b598411aa0e3f2385e3664c5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/3ab3f747b598411aa0e3f2385e3664c5 2024-11-19T17:50:45,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/3ab3f747b598411aa0e3f2385e3664c5, entries=200, sequenceid=118, filesize=38.6 K 2024-11-19T17:50:45,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/fb2a7876d296427eb8f6e99ef00d9de5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/fb2a7876d296427eb8f6e99ef00d9de5 2024-11-19T17:50:45,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/fb2a7876d296427eb8f6e99ef00d9de5, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T17:50:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/98df6ef2d25740b6aa9637fd9744b2d3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/98df6ef2d25740b6aa9637fd9744b2d3 2024-11-19T17:50:45,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/98df6ef2d25740b6aa9637fd9744b2d3, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T17:50:45,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b9c37934b4aa3330f167e8a793f13680 in 501ms, sequenceid=118, compaction requested=true 2024-11-19T17:50:45,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:45,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:45,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:45,112 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:45,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:45,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:45,112 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:45,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:45,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:45,114 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:45,114 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:50:45,114 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:45,114 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/7c82b62b35c546cd83f2bbdc5b88b48b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/dfd1e7f607d04abd9f8a74c12c7938ef, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/196f32edaa00460b9ecd1a3d4e9541c8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/3ab3f747b598411aa0e3f2385e3664c5] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=137.8 K 2024-11-19T17:50:45,114 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:45,114 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/7c82b62b35c546cd83f2bbdc5b88b48b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/dfd1e7f607d04abd9f8a74c12c7938ef, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/196f32edaa00460b9ecd1a3d4e9541c8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/3ab3f747b598411aa0e3f2385e3664c5] 2024-11-19T17:50:45,115 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c82b62b35c546cd83f2bbdc5b88b48b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732038639777 2024-11-19T17:50:45,115 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:45,115 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfd1e7f607d04abd9f8a74c12c7938ef, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732038640431 2024-11-19T17:50:45,115 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:50:45,115 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:45,115 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/80781c5431484c679d84c5ec9aafb33f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/39b8818c16f44206bd6edcb359772735, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1e4f9f851e72451da7e9598561f9e555, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/fb2a7876d296427eb8f6e99ef00d9de5] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=47.0 K 2024-11-19T17:50:45,115 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 80781c5431484c679d84c5ec9aafb33f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732038639777 2024-11-19T17:50:45,115 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 196f32edaa00460b9ecd1a3d4e9541c8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038641822 2024-11-19T17:50:45,116 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b8818c16f44206bd6edcb359772735, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732038640437 2024-11-19T17:50:45,116 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ab3f747b598411aa0e3f2385e3664c5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732038644063 2024-11-19T17:50:45,116 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4f9f851e72451da7e9598561f9e555, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038641822 2024-11-19T17:50:45,116 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2a7876d296427eb8f6e99ef00d9de5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732038644074 2024-11-19T17:50:45,124 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:45,124 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:45,125 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/3cf871ded25d40599edc1ea64743cd86 is 50, key is test_row_0/B:col10/1732038644074/Put/seqid=0 2024-11-19T17:50:45,126 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411190b22935d377c421ba3eda84a46aced45_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:45,130 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411190b22935d377c421ba3eda84a46aced45_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:45,130 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190b22935d377c421ba3eda84a46aced45_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:45,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742399_1575 (size=12241) 2024-11-19T17:50:45,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742400_1576 (size=4469) 2024-11-19T17:50:45,140 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#490 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:45,141 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/11ce7f8965ec4500bdafe27cacbbec8b is 175, key is test_row_0/A:col10/1732038644074/Put/seqid=0 2024-11-19T17:50:45,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742401_1577 (size=31195) 2024-11-19T17:50:45,148 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/11ce7f8965ec4500bdafe27cacbbec8b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/11ce7f8965ec4500bdafe27cacbbec8b 2024-11-19T17:50:45,152 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into 11ce7f8965ec4500bdafe27cacbbec8b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:45,152 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:45,153 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=12, startTime=1732038645112; duration=0sec 2024-11-19T17:50:45,153 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:45,153 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:50:45,153 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:50:45,154 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:50:45,154 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:50:45,154 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:45,154 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/7b5dc31aa7cf48b7930a6ce0b220aee3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f578cc44c69e442ca7077102d5ff1d54, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/94fd63105abf4f17b4f228065bad3a4a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/98df6ef2d25740b6aa9637fd9744b2d3] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=47.0 K 2024-11-19T17:50:45,154 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5dc31aa7cf48b7930a6ce0b220aee3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732038639777 2024-11-19T17:50:45,155 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting f578cc44c69e442ca7077102d5ff1d54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732038640437 2024-11-19T17:50:45,155 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94fd63105abf4f17b4f228065bad3a4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732038641822 2024-11-19T17:50:45,155 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98df6ef2d25740b6aa9637fd9744b2d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732038644074 2024-11-19T17:50:45,162 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:45,162 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/1efa8376aca347f3aed015795b654076 is 50, key is test_row_0/C:col10/1732038644074/Put/seqid=0 2024-11-19T17:50:45,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742402_1578 (size=12241) 2024-11-19T17:50:45,170 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/1efa8376aca347f3aed015795b654076 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1efa8376aca347f3aed015795b654076 2024-11-19T17:50:45,174 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into 1efa8376aca347f3aed015795b654076(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:45,174 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:45,174 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=12, startTime=1732038645112; duration=0sec 2024-11-19T17:50:45,174 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:45,174 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:50:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:45,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:45,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T17:50:45,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194afef87a9922469dbe8bfd71be3f5b5e_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038645230/Put/seqid=0 2024-11-19T17:50:45,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742403_1579 (size=12254) 2024-11-19T17:50:45,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:45,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194afef87a9922469dbe8bfd71be3f5b5e_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194afef87a9922469dbe8bfd71be3f5b5e_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:45,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/8d07b48006c6429fbc996a8da126e8c2, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:45,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/8d07b48006c6429fbc996a8da126e8c2 is 175, key is test_row_0/A:col10/1732038645230/Put/seqid=0 2024-11-19T17:50:45,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742404_1580 (size=31055) 2024-11-19T17:50:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038705286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038705286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038705287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038705288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038705292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038705399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038705399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038705399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038705399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038705408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,540 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/3cf871ded25d40599edc1ea64743cd86 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/3cf871ded25d40599edc1ea64743cd86 2024-11-19T17:50:45,544 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into 3cf871ded25d40599edc1ea64743cd86(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:45,544 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:45,544 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=12, startTime=1732038645112; duration=0sec 2024-11-19T17:50:45,544 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:45,544 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:50:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038705609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038705609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038705609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038705609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038705618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/8d07b48006c6429fbc996a8da126e8c2 2024-11-19T17:50:45,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/6a039118874b41ada5744bc28cf16b19 is 50, key is test_row_0/B:col10/1732038645230/Put/seqid=0 2024-11-19T17:50:45,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742405_1581 (size=12101) 2024-11-19T17:50:45,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/6a039118874b41ada5744bc28cf16b19 2024-11-19T17:50:45,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/f92871bcd06d43599e692f4df3b44789 is 50, key is test_row_0/C:col10/1732038645230/Put/seqid=0 2024-11-19T17:50:45,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742406_1582 (size=12101) 2024-11-19T17:50:45,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/f92871bcd06d43599e692f4df3b44789 2024-11-19T17:50:45,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/8d07b48006c6429fbc996a8da126e8c2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/8d07b48006c6429fbc996a8da126e8c2 2024-11-19T17:50:45,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/8d07b48006c6429fbc996a8da126e8c2, entries=150, sequenceid=133, filesize=30.3 K 2024-11-19T17:50:45,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/6a039118874b41ada5744bc28cf16b19 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/6a039118874b41ada5744bc28cf16b19 2024-11-19T17:50:45,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/6a039118874b41ada5744bc28cf16b19, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:50:45,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/f92871bcd06d43599e692f4df3b44789 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f92871bcd06d43599e692f4df3b44789 2024-11-19T17:50:45,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f92871bcd06d43599e692f4df3b44789, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T17:50:45,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b9c37934b4aa3330f167e8a793f13680 in 487ms, sequenceid=133, compaction requested=false 2024-11-19T17:50:45,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T17:50:45,909 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:45,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-19T17:50:45,911 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T17:50:45,912 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:45,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:45,921 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:45,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T17:50:45,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198ef177c203234b4d965488367363d3f0_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038645290/Put/seqid=0 2024-11-19T17:50:45,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038705928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742407_1583 (size=17284) 2024-11-19T17:50:45,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038705929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038705934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038705936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038705936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T17:50:46,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038706038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038706049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038706049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038706049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T17:50:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038706249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038706259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038706259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038706259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,336 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:46,345 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198ef177c203234b4d965488367363d3f0_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198ef177c203234b4d965488367363d3f0_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:46,346 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/324523b87c354f6083754ed351f5cc9c, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:46,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/324523b87c354f6083754ed351f5cc9c is 175, key is test_row_0/A:col10/1732038645290/Put/seqid=0 2024-11-19T17:50:46,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742408_1584 (size=48389) 2024-11-19T17:50:46,355 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/324523b87c354f6083754ed351f5cc9c 2024-11-19T17:50:46,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/985f4c32e7654a66ba16c98ef606cf38 is 50, key is test_row_0/B:col10/1732038645290/Put/seqid=0 2024-11-19T17:50:46,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742409_1585 (size=12151) 2024-11-19T17:50:46,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:46,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:46,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038706449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T17:50:46,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038706558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038706568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038706568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038706568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:46,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:46,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:46,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/985f4c32e7654a66ba16c98ef606cf38 2024-11-19T17:50:46,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/3f0e0d06122843649bdf6f36af270964 is 50, key is test_row_0/C:col10/1732038645290/Put/seqid=0 2024-11-19T17:50:46,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742410_1586 (size=12151) 2024-11-19T17:50:46,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:46,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T17:50:47,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038707068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038707080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038707080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:47,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038707083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:47,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:47,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/3f0e0d06122843649bdf6f36af270964 2024-11-19T17:50:47,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/324523b87c354f6083754ed351f5cc9c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/324523b87c354f6083754ed351f5cc9c 2024-11-19T17:50:47,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/324523b87c354f6083754ed351f5cc9c, entries=250, sequenceid=159, filesize=47.3 K 2024-11-19T17:50:47,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/985f4c32e7654a66ba16c98ef606cf38 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/985f4c32e7654a66ba16c98ef606cf38 2024-11-19T17:50:47,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/985f4c32e7654a66ba16c98ef606cf38, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T17:50:47,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/3f0e0d06122843649bdf6f36af270964 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3f0e0d06122843649bdf6f36af270964 2024-11-19T17:50:47,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3f0e0d06122843649bdf6f36af270964, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T17:50:47,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b9c37934b4aa3330f167e8a793f13680 in 1278ms, sequenceid=159, compaction requested=true 2024-11-19T17:50:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:50:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:47,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T17:50:47,199 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:47,199 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:47,200 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:47,200 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:47,200 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:50:47,200 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:50:47,200 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:47,200 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:47,200 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1efa8376aca347f3aed015795b654076, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f92871bcd06d43599e692f4df3b44789, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3f0e0d06122843649bdf6f36af270964] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=35.6 K 2024-11-19T17:50:47,200 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/11ce7f8965ec4500bdafe27cacbbec8b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/8d07b48006c6429fbc996a8da126e8c2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/324523b87c354f6083754ed351f5cc9c] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=108.0 K 2024-11-19T17:50:47,201 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:47,201 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/11ce7f8965ec4500bdafe27cacbbec8b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/8d07b48006c6429fbc996a8da126e8c2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/324523b87c354f6083754ed351f5cc9c] 2024-11-19T17:50:47,201 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ce7f8965ec4500bdafe27cacbbec8b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732038644074 2024-11-19T17:50:47,201 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1efa8376aca347f3aed015795b654076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732038644074 2024-11-19T17:50:47,201 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting f92871bcd06d43599e692f4df3b44789, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038644616 2024-11-19T17:50:47,201 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d07b48006c6429fbc996a8da126e8c2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038644616 2024-11-19T17:50:47,202 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0e0d06122843649bdf6f36af270964, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038645290 2024-11-19T17:50:47,203 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 324523b87c354f6083754ed351f5cc9c, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038645284 2024-11-19T17:50:47,232 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:47,234 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:47,235 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/dcc5ce986e6a40669d5a9a4bdf9460b2 is 50, key is test_row_0/C:col10/1732038645290/Put/seqid=0 2024-11-19T17:50:47,240 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411196012afaf3b594a61b60de46a09db4002_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:47,242 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411196012afaf3b594a61b60de46a09db4002_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:47,242 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196012afaf3b594a61b60de46a09db4002_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:47,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742411_1587 (size=12493) 2024-11-19T17:50:47,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742412_1588 (size=4469) 2024-11-19T17:50:47,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T17:50:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:47,289 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T17:50:47,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111955573ecf7690404dbd6ce28469fe37fa_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038645933/Put/seqid=0 2024-11-19T17:50:47,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742413_1589 (size=12304) 2024-11-19T17:50:47,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:47,320 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111955573ecf7690404dbd6ce28469fe37fa_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111955573ecf7690404dbd6ce28469fe37fa_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:47,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/23bc0aae60d34d5d8ed4945575946502, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/23bc0aae60d34d5d8ed4945575946502 is 175, key is test_row_0/A:col10/1732038645933/Put/seqid=0 2024-11-19T17:50:47,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742414_1590 (size=31105) 2024-11-19T17:50:47,336 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/23bc0aae60d34d5d8ed4945575946502 2024-11-19T17:50:47,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/30a101c03c8a4fd5b760b702f3bc44c2 is 50, key is test_row_0/B:col10/1732038645933/Put/seqid=0 2024-11-19T17:50:47,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742415_1591 (size=12151) 2024-11-19T17:50:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:47,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038707576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:47,657 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/dcc5ce986e6a40669d5a9a4bdf9460b2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/dcc5ce986e6a40669d5a9a4bdf9460b2 2024-11-19T17:50:47,658 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#499 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:47,658 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/a93bb4d071c54ebb800ad8821ce6a0b3 is 175, key is test_row_0/A:col10/1732038645290/Put/seqid=0 2024-11-19T17:50:47,667 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into dcc5ce986e6a40669d5a9a4bdf9460b2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:47,667 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:47,667 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=13, startTime=1732038647199; duration=0sec 2024-11-19T17:50:47,667 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:47,667 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:50:47,667 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:47,668 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:47,668 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:50:47,668 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:47,668 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/3cf871ded25d40599edc1ea64743cd86, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/6a039118874b41ada5744bc28cf16b19, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/985f4c32e7654a66ba16c98ef606cf38] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=35.6 K 2024-11-19T17:50:47,669 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf871ded25d40599edc1ea64743cd86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732038644074 2024-11-19T17:50:47,669 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a039118874b41ada5744bc28cf16b19, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732038644616 2024-11-19T17:50:47,669 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 985f4c32e7654a66ba16c98ef606cf38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038645290 2024-11-19T17:50:47,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742416_1592 (size=31447) 2024-11-19T17:50:47,678 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/a93bb4d071c54ebb800ad8821ce6a0b3 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/a93bb4d071c54ebb800ad8821ce6a0b3 2024-11-19T17:50:47,682 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into a93bb4d071c54ebb800ad8821ce6a0b3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:47,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:47,682 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=13, startTime=1732038647199; duration=0sec 2024-11-19T17:50:47,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:47,682 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:50:47,685 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:47,685 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/2cd2d97f7a8940289e7fb2d938637927 is 50, key is test_row_0/B:col10/1732038645290/Put/seqid=0 2024-11-19T17:50:47,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742417_1593 (size=12493) 2024-11-19T17:50:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038707688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:47,748 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/30a101c03c8a4fd5b760b702f3bc44c2 2024-11-19T17:50:47,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/a52fa0f4390b47d08032882e3846b265 is 50, key is test_row_0/C:col10/1732038645933/Put/seqid=0 2024-11-19T17:50:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742418_1594 (size=12151) 2024-11-19T17:50:47,766 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/a52fa0f4390b47d08032882e3846b265 2024-11-19T17:50:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/23bc0aae60d34d5d8ed4945575946502 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/23bc0aae60d34d5d8ed4945575946502 2024-11-19T17:50:47,773 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/23bc0aae60d34d5d8ed4945575946502, entries=150, sequenceid=170, filesize=30.4 K 2024-11-19T17:50:47,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/30a101c03c8a4fd5b760b702f3bc44c2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/30a101c03c8a4fd5b760b702f3bc44c2 2024-11-19T17:50:47,780 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/30a101c03c8a4fd5b760b702f3bc44c2, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T17:50:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/a52fa0f4390b47d08032882e3846b265 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/a52fa0f4390b47d08032882e3846b265 2024-11-19T17:50:47,785 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/a52fa0f4390b47d08032882e3846b265, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T17:50:47,786 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b9c37934b4aa3330f167e8a793f13680 in 497ms, sequenceid=170, compaction requested=false 2024-11-19T17:50:47,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:47,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:47,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-19T17:50:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-19T17:50:47,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T17:50:47,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8760 sec 2024-11-19T17:50:47,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8860 sec 2024-11-19T17:50:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:47,912 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:47,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T17:50:47,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111940d4976b90484f85a47f2a2e4db5b69f_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038647567/Put/seqid=0 2024-11-19T17:50:47,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742419_1595 (size=14794) 2024-11-19T17:50:47,924 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:47,930 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111940d4976b90484f85a47f2a2e4db5b69f_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111940d4976b90484f85a47f2a2e4db5b69f_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:47,931 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/55ded3ec8ea84c709994d783dfea208f, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:47,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/55ded3ec8ea84c709994d783dfea208f is 175, key is test_row_0/A:col10/1732038647567/Put/seqid=0 2024-11-19T17:50:47,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742420_1596 (size=39749) 2024-11-19T17:50:47,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038707951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038708059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T17:50:48,069 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:48,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-19T17:50:48,071 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:50:48,072 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:48,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:48,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038708080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:48,081 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038708088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:48,093 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,095 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/2cd2d97f7a8940289e7fb2d938637927 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/2cd2d97f7a8940289e7fb2d938637927 2024-11-19T17:50:48,100 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into 2cd2d97f7a8940289e7fb2d938637927(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:48,100 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:48,100 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=13, startTime=1732038647199; duration=0sec 2024-11-19T17:50:48,100 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:48,100 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:50:48,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038708099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:48,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038708099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:48,102 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,102 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:50:48,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T17:50:48,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038708269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,347 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/55ded3ec8ea84c709994d783dfea208f 2024-11-19T17:50:48,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/55a4e3fbaab041b291a2ef3a082dc168 is 50, key is test_row_0/B:col10/1732038647567/Put/seqid=0 2024-11-19T17:50:48,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742421_1597 (size=12151) 2024-11-19T17:50:48,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/55a4e3fbaab041b291a2ef3a082dc168 2024-11-19T17:50:48,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/ae9c016c488e4e0cbe44389aa7129957 is 50, key is test_row_0/C:col10/1732038647567/Put/seqid=0 2024-11-19T17:50:48,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T17:50:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742422_1598 (size=12151) 2024-11-19T17:50:48,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/ae9c016c488e4e0cbe44389aa7129957 2024-11-19T17:50:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:50:48,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/55ded3ec8ea84c709994d783dfea208f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/55ded3ec8ea84c709994d783dfea208f 2024-11-19T17:50:48,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/55ded3ec8ea84c709994d783dfea208f, entries=200, sequenceid=198, filesize=38.8 K 2024-11-19T17:50:48,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/55a4e3fbaab041b291a2ef3a082dc168 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/55a4e3fbaab041b291a2ef3a082dc168 2024-11-19T17:50:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/55a4e3fbaab041b291a2ef3a082dc168, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T17:50:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/ae9c016c488e4e0cbe44389aa7129957 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ae9c016c488e4e0cbe44389aa7129957 2024-11-19T17:50:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ae9c016c488e4e0cbe44389aa7129957, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T17:50:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for b9c37934b4aa3330f167e8a793f13680 in 501ms, sequenceid=198, compaction requested=true 2024-11-19T17:50:48,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:48,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:48,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:48,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:50:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:48,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T17:50:48,413 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:48,413 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,414 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:48,414 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:50:48,415 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,415 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/a93bb4d071c54ebb800ad8821ce6a0b3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/23bc0aae60d34d5d8ed4945575946502, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/55ded3ec8ea84c709994d783dfea208f] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=99.9 K 2024-11-19T17:50:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,415 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,415 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/a93bb4d071c54ebb800ad8821ce6a0b3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/23bc0aae60d34d5d8ed4945575946502, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/55ded3ec8ea84c709994d783dfea208f] 2024-11-19T17:50:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,416 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,416 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting a93bb4d071c54ebb800ad8821ce6a0b3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038645290 2024-11-19T17:50:48,416 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:50:48,416 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,416 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/dcc5ce986e6a40669d5a9a4bdf9460b2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/a52fa0f4390b47d08032882e3846b265, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ae9c016c488e4e0cbe44389aa7129957] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=35.9 K 2024-11-19T17:50:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,417 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc5ce986e6a40669d5a9a4bdf9460b2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038645290 2024-11-19T17:50:48,417 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23bc0aae60d34d5d8ed4945575946502, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732038645932 2024-11-19T17:50:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,417 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting a52fa0f4390b47d08032882e3846b265, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732038645932 2024-11-19T17:50:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,418 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ded3ec8ea84c709994d783dfea208f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038647556 2024-11-19T17:50:48,418 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ae9c016c488e4e0cbe44389aa7129957, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038647567 2024-11-19T17:50:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,422 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:48,424 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,425 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/6a3652b1f6e6489b89e419fcc4944fcd is 50, key is test_row_0/C:col10/1732038647567/Put/seqid=0 2024-11-19T17:50:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,425 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b954aeb7622444ae87f10361f0e92410_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,427 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b954aeb7622444ae87f10361f0e92410_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:48,427 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b954aeb7622444ae87f10361f0e92410_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742423_1599 (size=12595) 2024-11-19T17:50:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742424_1600 (size=4469) 2024-11-19T17:50:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,438 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#507 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,438 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/631022858eb84a9da1bf9be2e9d4c982 is 175, key is test_row_0/A:col10/1732038647567/Put/seqid=0 2024-11-19T17:50:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,448 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/6a3652b1f6e6489b89e419fcc4944fcd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/6a3652b1f6e6489b89e419fcc4944fcd 2024-11-19T17:50:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742425_1601 (size=31549) 2024-11-19T17:50:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,459 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into 6a3652b1f6e6489b89e419fcc4944fcd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,459 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:48,459 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=13, startTime=1732038648413; duration=0sec 2024-11-19T17:50:48,459 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:48,459 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:50:48,459 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,461 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,461 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:50:48,461 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,461 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/2cd2d97f7a8940289e7fb2d938637927, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/30a101c03c8a4fd5b760b702f3bc44c2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/55a4e3fbaab041b291a2ef3a082dc168] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=35.9 K 2024-11-19T17:50:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,462 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd2d97f7a8940289e7fb2d938637927, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732038645290 2024-11-19T17:50:48,462 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 30a101c03c8a4fd5b760b702f3bc44c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732038645932 2024-11-19T17:50:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,463 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a4e3fbaab041b291a2ef3a082dc168, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038647567 2024-11-19T17:50:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,473 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,473 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/c25252df63a342d0b5639cc106a6cd6b is 50, key is test_row_0/B:col10/1732038647567/Put/seqid=0 2024-11-19T17:50:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742426_1602 (size=12595) 2024-11-19T17:50:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T17:50:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:48,534 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T17:50:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111920974a04e5874387b15529a32b538bf0_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038647950/Put/seqid=0 2024-11-19T17:50:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742427_1603 (size=9814) 2024-11-19T17:50:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:48,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:50:48,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038708713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038708818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:48,854 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/631022858eb84a9da1bf9be2e9d4c982 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/631022858eb84a9da1bf9be2e9d4c982 2024-11-19T17:50:48,859 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into 631022858eb84a9da1bf9be2e9d4c982(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:48,859 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:48,859 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=13, startTime=1732038648413; duration=0sec 2024-11-19T17:50:48,859 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:48,859 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:50:48,897 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/c25252df63a342d0b5639cc106a6cd6b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c25252df63a342d0b5639cc106a6cd6b 2024-11-19T17:50:48,902 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into c25252df63a342d0b5639cc106a6cd6b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:48,902 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:48,902 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=13, startTime=1732038648413; duration=0sec 2024-11-19T17:50:48,902 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:48,902 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:50:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:48,956 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111920974a04e5874387b15529a32b538bf0_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111920974a04e5874387b15529a32b538bf0_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b859191799464161bad0460f9ba1b578, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b859191799464161bad0460f9ba1b578 is 175, key is test_row_0/A:col10/1732038647950/Put/seqid=0 2024-11-19T17:50:48,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742428_1604 (size=22461) 2024-11-19T17:50:48,971 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b859191799464161bad0460f9ba1b578 2024-11-19T17:50:48,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/76db30ee062940c89cdb42f69708b3ff is 50, key is test_row_0/B:col10/1732038647950/Put/seqid=0 2024-11-19T17:50:48,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742429_1605 (size=9757) 2024-11-19T17:50:49,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038709028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:50:49,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038709339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:49,394 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/76db30ee062940c89cdb42f69708b3ff 2024-11-19T17:50:49,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/2e27ac4146fd4d24828cb3a0b93cee76 is 50, key is test_row_0/C:col10/1732038647950/Put/seqid=0 2024-11-19T17:50:49,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742430_1606 (size=9757) 2024-11-19T17:50:49,814 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/2e27ac4146fd4d24828cb3a0b93cee76 2024-11-19T17:50:49,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b859191799464161bad0460f9ba1b578 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b859191799464161bad0460f9ba1b578 2024-11-19T17:50:49,822 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b859191799464161bad0460f9ba1b578, entries=100, sequenceid=210, filesize=21.9 K 2024-11-19T17:50:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/76db30ee062940c89cdb42f69708b3ff as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/76db30ee062940c89cdb42f69708b3ff 2024-11-19T17:50:49,830 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/76db30ee062940c89cdb42f69708b3ff, entries=100, sequenceid=210, filesize=9.5 K 2024-11-19T17:50:49,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/2e27ac4146fd4d24828cb3a0b93cee76 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2e27ac4146fd4d24828cb3a0b93cee76 2024-11-19T17:50:49,834 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2e27ac4146fd4d24828cb3a0b93cee76, entries=100, sequenceid=210, filesize=9.5 K 2024-11-19T17:50:49,835 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for b9c37934b4aa3330f167e8a793f13680 in 1301ms, sequenceid=210, compaction requested=false 2024-11-19T17:50:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-19T17:50:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-19T17:50:49,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T17:50:49,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7640 sec 2024-11-19T17:50:49,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7700 sec 2024-11-19T17:50:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:49,852 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:49,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T17:50:49,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193493b929f7374db08055ff282595953f_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038648710/Put/seqid=0 2024-11-19T17:50:49,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742431_1607 (size=12304) 2024-11-19T17:50:49,868 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:49,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193493b929f7374db08055ff282595953f_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193493b929f7374db08055ff282595953f_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:49,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/2ab7813d7a2241928859330795667739, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:49,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/2ab7813d7a2241928859330795667739 is 175, key is test_row_0/A:col10/1732038648710/Put/seqid=0 2024-11-19T17:50:49,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038709877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:49,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742432_1608 (size=31105) 2024-11-19T17:50:49,883 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/2ab7813d7a2241928859330795667739 2024-11-19T17:50:49,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/1eea4ff96930495d88b341c637886315 is 50, key is test_row_0/B:col10/1732038648710/Put/seqid=0 2024-11-19T17:50:49,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742433_1609 (size=12151) 2024-11-19T17:50:49,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/1eea4ff96930495d88b341c637886315 2024-11-19T17:50:49,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/99482849796b4e6f8bb4241a564dd845 is 50, key is test_row_0/C:col10/1732038648710/Put/seqid=0 2024-11-19T17:50:49,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742434_1610 (size=12151) 2024-11-19T17:50:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038709989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038710099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:50,101 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:50,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038710109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:50,112 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_2' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:50,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038710118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:50,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038710119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:50,122 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:50,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:50,123 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038710198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:50:50,219 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:50,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-19T17:50:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:50:50,223 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:50,223 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:50,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:50:50,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/99482849796b4e6f8bb4241a564dd845 2024-11-19T17:50:50,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/2ab7813d7a2241928859330795667739 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2ab7813d7a2241928859330795667739 2024-11-19T17:50:50,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2ab7813d7a2241928859330795667739, entries=150, sequenceid=239, filesize=30.4 K 2024-11-19T17:50:50,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/1eea4ff96930495d88b341c637886315 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1eea4ff96930495d88b341c637886315 2024-11-19T17:50:50,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1eea4ff96930495d88b341c637886315, entries=150, sequenceid=239, filesize=11.9 K 2024-11-19T17:50:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/99482849796b4e6f8bb4241a564dd845 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/99482849796b4e6f8bb4241a564dd845 2024-11-19T17:50:50,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/99482849796b4e6f8bb4241a564dd845, entries=150, sequenceid=239, filesize=11.9 K 2024-11-19T17:50:50,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b9c37934b4aa3330f167e8a793f13680 in 503ms, sequenceid=239, compaction requested=true 2024-11-19T17:50:50,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:50,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:50,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:50,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:50,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:50:50,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:50,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T17:50:50,356 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:50,356 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:50,357 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:50,357 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:50:50,357 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:50,357 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/6a3652b1f6e6489b89e419fcc4944fcd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2e27ac4146fd4d24828cb3a0b93cee76, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/99482849796b4e6f8bb4241a564dd845] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=33.7 K 2024-11-19T17:50:50,357 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:50,357 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:50:50,357 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:50,357 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/631022858eb84a9da1bf9be2e9d4c982, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b859191799464161bad0460f9ba1b578, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2ab7813d7a2241928859330795667739] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=83.1 K 2024-11-19T17:50:50,357 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:50,357 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/631022858eb84a9da1bf9be2e9d4c982, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b859191799464161bad0460f9ba1b578, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2ab7813d7a2241928859330795667739] 2024-11-19T17:50:50,358 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3652b1f6e6489b89e419fcc4944fcd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038647567 2024-11-19T17:50:50,358 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631022858eb84a9da1bf9be2e9d4c982, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038647567 2024-11-19T17:50:50,358 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e27ac4146fd4d24828cb3a0b93cee76, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038647946 2024-11-19T17:50:50,358 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting b859191799464161bad0460f9ba1b578, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038647946 2024-11-19T17:50:50,358 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab7813d7a2241928859330795667739, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732038648685 2024-11-19T17:50:50,358 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 99482849796b4e6f8bb4241a564dd845, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732038648685 2024-11-19T17:50:50,366 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:50,366 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/ff2c254ab23d40ec84a4b65ce3b36b22 is 50, key is test_row_0/C:col10/1732038648710/Put/seqid=0 2024-11-19T17:50:50,369 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:50,371 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191a0a6e4f851345118f0121767aee5d1a_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:50,373 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191a0a6e4f851345118f0121767aee5d1a_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:50,373 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191a0a6e4f851345118f0121767aee5d1a_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:50,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T17:50:50,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:50,377 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-19T17:50:50,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742435_1611 (size=12697) 2024-11-19T17:50:50,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742436_1612 (size=4469) 2024-11-19T17:50:50,391 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#517 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:50,392 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/1d9f38bfc3704b6f9053a73bcb55523e is 175, key is test_row_0/A:col10/1732038648710/Put/seqid=0 2024-11-19T17:50:50,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742437_1613 (size=31651) 2024-11-19T17:50:50,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111994083457f9c54d3f90737d1d3a7c03fb_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038649871/Put/seqid=0 2024-11-19T17:50:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742438_1614 (size=12304) 2024-11-19T17:50:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:50,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:50:50,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038710626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:50,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038710738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:50,778 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/ff2c254ab23d40ec84a4b65ce3b36b22 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ff2c254ab23d40ec84a4b65ce3b36b22 2024-11-19T17:50:50,785 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into ff2c254ab23d40ec84a4b65ce3b36b22(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:50,785 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:50,785 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=13, startTime=1732038650355; duration=0sec 2024-11-19T17:50:50,785 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:50,785 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:50:50,785 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:50,787 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:50,787 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:50:50,787 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:50,787 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c25252df63a342d0b5639cc106a6cd6b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/76db30ee062940c89cdb42f69708b3ff, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1eea4ff96930495d88b341c637886315] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=33.7 K 2024-11-19T17:50:50,787 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c25252df63a342d0b5639cc106a6cd6b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732038647567 2024-11-19T17:50:50,788 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 76db30ee062940c89cdb42f69708b3ff, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732038647946 2024-11-19T17:50:50,788 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eea4ff96930495d88b341c637886315, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732038648685 2024-11-19T17:50:50,793 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:50,794 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/af76a794d0ac4574824839a55a5dba27 is 50, key is test_row_0/B:col10/1732038648710/Put/seqid=0 2024-11-19T17:50:50,814 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/1d9f38bfc3704b6f9053a73bcb55523e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/1d9f38bfc3704b6f9053a73bcb55523e 2024-11-19T17:50:50,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742439_1615 (size=12697) 2024-11-19T17:50:50,822 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into 1d9f38bfc3704b6f9053a73bcb55523e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:50,822 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:50,822 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=13, startTime=1732038650355; duration=0sec 2024-11-19T17:50:50,822 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:50,822 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:50:50,826 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/af76a794d0ac4574824839a55a5dba27 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/af76a794d0ac4574824839a55a5dba27 2024-11-19T17:50:50,831 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into af76a794d0ac4574824839a55a5dba27(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:50,831 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:50,831 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=13, startTime=1732038650355; duration=0sec 2024-11-19T17:50:50,831 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:50,831 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:50:50,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:50:50,849 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111994083457f9c54d3f90737d1d3a7c03fb_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994083457f9c54d3f90737d1d3a7c03fb_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/cb0240f4f66c4e519348a3e3400fd161, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/cb0240f4f66c4e519348a3e3400fd161 is 175, key is test_row_0/A:col10/1732038649871/Put/seqid=0 2024-11-19T17:50:50,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742440_1616 (size=31105) 2024-11-19T17:50:50,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038710951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:51,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038711259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:51,265 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/cb0240f4f66c4e519348a3e3400fd161 2024-11-19T17:50:51,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/342eee0557e2424a945137ffb08f02f2 is 50, key is test_row_0/B:col10/1732038649871/Put/seqid=0 2024-11-19T17:50:51,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742441_1617 (size=12151) 2024-11-19T17:50:51,274 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/342eee0557e2424a945137ffb08f02f2 2024-11-19T17:50:51,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/3151a350fdc444f6aaa4a9eed30486e6 is 50, key is test_row_0/C:col10/1732038649871/Put/seqid=0 2024-11-19T17:50:51,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742442_1618 (size=12151) 2024-11-19T17:50:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:50:51,690 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/3151a350fdc444f6aaa4a9eed30486e6 2024-11-19T17:50:51,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/cb0240f4f66c4e519348a3e3400fd161 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/cb0240f4f66c4e519348a3e3400fd161 2024-11-19T17:50:51,698 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/cb0240f4f66c4e519348a3e3400fd161, entries=150, sequenceid=248, filesize=30.4 K 2024-11-19T17:50:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/342eee0557e2424a945137ffb08f02f2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/342eee0557e2424a945137ffb08f02f2 2024-11-19T17:50:51,703 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/342eee0557e2424a945137ffb08f02f2, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T17:50:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/3151a350fdc444f6aaa4a9eed30486e6 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3151a350fdc444f6aaa4a9eed30486e6 2024-11-19T17:50:51,708 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3151a350fdc444f6aaa4a9eed30486e6, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T17:50:51,709 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for b9c37934b4aa3330f167e8a793f13680 in 1333ms, sequenceid=248, compaction requested=false 2024-11-19T17:50:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-19T17:50:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-19T17:50:51,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T17:50:51,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4890 sec 2024-11-19T17:50:51,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4960 sec 2024-11-19T17:50:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:51,770 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:51,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-19T17:50:51,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a92c5524a0a642ffbb9f1889315750d7_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038650619/Put/seqid=0 2024-11-19T17:50:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038711792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:51,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742443_1619 (size=14994) 2024-11-19T17:50:51,798 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:51,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a92c5524a0a642ffbb9f1889315750d7_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a92c5524a0a642ffbb9f1889315750d7_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:51,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/9cb157caaf754b65885f8c8cc8dd9359, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:51,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/9cb157caaf754b65885f8c8cc8dd9359 is 175, key is test_row_0/A:col10/1732038650619/Put/seqid=0 2024-11-19T17:50:51,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742444_1620 (size=39949) 2024-11-19T17:50:51,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038711899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:52,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038712109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:52,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/9cb157caaf754b65885f8c8cc8dd9359 2024-11-19T17:50:52,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/621e557b54ec45e7bae23ac1c9b98fad is 50, key is test_row_0/B:col10/1732038650619/Put/seqid=0 2024-11-19T17:50:52,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742445_1621 (size=12301) 2024-11-19T17:50:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:50:52,368 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:52,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-19T17:50:52,371 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:50:52,371 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:52,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038712419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:50:52,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T17:50:52,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/621e557b54ec45e7bae23ac1c9b98fad 2024-11-19T17:50:52,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/65f6f9918dc047129adb1124376d5b73 is 50, key is test_row_0/C:col10/1732038650619/Put/seqid=0 2024-11-19T17:50:52,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742446_1622 (size=12301) 2024-11-19T17:50:52,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/65f6f9918dc047129adb1124376d5b73 2024-11-19T17:50:52,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/9cb157caaf754b65885f8c8cc8dd9359 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/9cb157caaf754b65885f8c8cc8dd9359 2024-11-19T17:50:52,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/9cb157caaf754b65885f8c8cc8dd9359, entries=200, sequenceid=279, filesize=39.0 K 2024-11-19T17:50:52,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/621e557b54ec45e7bae23ac1c9b98fad as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/621e557b54ec45e7bae23ac1c9b98fad 2024-11-19T17:50:52,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T17:50:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/621e557b54ec45e7bae23ac1c9b98fad, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T17:50:52,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/65f6f9918dc047129adb1124376d5b73 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/65f6f9918dc047129adb1124376d5b73 2024-11-19T17:50:52,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/65f6f9918dc047129adb1124376d5b73, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T17:50:52,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b9c37934b4aa3330f167e8a793f13680 in 916ms, sequenceid=279, compaction requested=true 2024-11-19T17:50:52,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:52,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:52,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:52,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:52,687 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:52,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:52,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:52,687 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:52,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:52,687 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:52,688 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:52,688 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:50:52,688 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:50:52,688 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,688 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/1d9f38bfc3704b6f9053a73bcb55523e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/cb0240f4f66c4e519348a3e3400fd161, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/9cb157caaf754b65885f8c8cc8dd9359] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=100.3 K 2024-11-19T17:50:52,688 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,688 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/1d9f38bfc3704b6f9053a73bcb55523e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/cb0240f4f66c4e519348a3e3400fd161, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/9cb157caaf754b65885f8c8cc8dd9359] 2024-11-19T17:50:52,688 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,688 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/af76a794d0ac4574824839a55a5dba27, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/342eee0557e2424a945137ffb08f02f2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/621e557b54ec45e7bae23ac1c9b98fad] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=36.3 K 2024-11-19T17:50:52,688 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting af76a794d0ac4574824839a55a5dba27, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732038648685 2024-11-19T17:50:52,688 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d9f38bfc3704b6f9053a73bcb55523e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732038648685 2024-11-19T17:50:52,689 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 342eee0557e2424a945137ffb08f02f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732038649867 2024-11-19T17:50:52,689 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb0240f4f66c4e519348a3e3400fd161, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732038649867 2024-11-19T17:50:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:50:52,690 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cb157caaf754b65885f8c8cc8dd9359, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732038650595 2024-11-19T17:50:52,690 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 621e557b54ec45e7bae23ac1c9b98fad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732038650595 2024-11-19T17:50:52,713 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:52,714 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:52,714 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/bc8febbeac2d4bf09c19f6f0b4a9e46f is 50, key is test_row_0/B:col10/1732038650619/Put/seqid=0 2024-11-19T17:50:52,726 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195cfaf48721cb4c1788912d83fab75169_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:52,728 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195cfaf48721cb4c1788912d83fab75169_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:52,728 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195cfaf48721cb4c1788912d83fab75169_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:52,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742447_1623 (size=12949) 2024-11-19T17:50:52,735 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/bc8febbeac2d4bf09c19f6f0b4a9e46f as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/bc8febbeac2d4bf09c19f6f0b4a9e46f 2024-11-19T17:50:52,739 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into bc8febbeac2d4bf09c19f6f0b4a9e46f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:52,739 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:52,739 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=13, startTime=1732038652686; duration=0sec 2024-11-19T17:50:52,739 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:52,739 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:50:52,739 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:52,740 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:52,741 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:50:52,741 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,741 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ff2c254ab23d40ec84a4b65ce3b36b22, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3151a350fdc444f6aaa4a9eed30486e6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/65f6f9918dc047129adb1124376d5b73] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=36.3 K 2024-11-19T17:50:52,741 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting ff2c254ab23d40ec84a4b65ce3b36b22, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732038648685 2024-11-19T17:50:52,741 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 3151a350fdc444f6aaa4a9eed30486e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732038649867 2024-11-19T17:50:52,741 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 65f6f9918dc047129adb1124376d5b73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732038650595 2024-11-19T17:50:52,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742448_1624 (size=4469) 2024-11-19T17:50:52,750 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:52,751 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/c2b7913b9fc54210aff16e513c0cbf69 is 50, key is test_row_0/C:col10/1732038650619/Put/seqid=0 2024-11-19T17:50:52,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742449_1625 (size=12949) 2024-11-19T17:50:52,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T17:50:52,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:52,829 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T17:50:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c23ef4bef4894ac6b4846cbef1736659_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038651787/Put/seqid=0 2024-11-19T17:50:52,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742450_1626 (size=12454) 2024-11-19T17:50:52,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:52,873 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c23ef4bef4894ac6b4846cbef1736659_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c23ef4bef4894ac6b4846cbef1736659_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:52,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/19880fcdf36844759880012bef0dde2c, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:52,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/19880fcdf36844759880012bef0dde2c is 175, key is test_row_0/A:col10/1732038651787/Put/seqid=0 2024-11-19T17:50:52,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742451_1627 (size=31255) 2024-11-19T17:50:52,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:50:53,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038713062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:53,144 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#526 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:53,144 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/90394742eaa54a95b8c33ae7b490039b is 175, key is test_row_0/A:col10/1732038650619/Put/seqid=0 2024-11-19T17:50:53,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742452_1628 (size=31903) 2024-11-19T17:50:53,157 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/c2b7913b9fc54210aff16e513c0cbf69 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/c2b7913b9fc54210aff16e513c0cbf69 2024-11-19T17:50:53,162 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into c2b7913b9fc54210aff16e513c0cbf69(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:53,162 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:53,162 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=13, startTime=1732038652687; duration=0sec 2024-11-19T17:50:53,162 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:53,162 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:50:53,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038713170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:53,286 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/19880fcdf36844759880012bef0dde2c 2024-11-19T17:50:53,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/eddf5f0da8564a9b81bbe9f25e3972fb is 50, key is test_row_0/B:col10/1732038651787/Put/seqid=0 2024-11-19T17:50:53,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742453_1629 (size=12301) 2024-11-19T17:50:53,306 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/eddf5f0da8564a9b81bbe9f25e3972fb 2024-11-19T17:50:53,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/522b0fd5064c4eff91fa56e2b982ed3d is 50, key is test_row_0/C:col10/1732038651787/Put/seqid=0 2024-11-19T17:50:53,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742454_1630 (size=12301) 2024-11-19T17:50:53,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038713378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:50:53,552 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/90394742eaa54a95b8c33ae7b490039b as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/90394742eaa54a95b8c33ae7b490039b 2024-11-19T17:50:53,556 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into 90394742eaa54a95b8c33ae7b490039b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:53,556 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:53,556 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=13, startTime=1732038652686; duration=0sec 2024-11-19T17:50:53,556 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:53,556 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:50:53,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038713689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:53,726 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/522b0fd5064c4eff91fa56e2b982ed3d 2024-11-19T17:50:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/19880fcdf36844759880012bef0dde2c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/19880fcdf36844759880012bef0dde2c 2024-11-19T17:50:53,731 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/19880fcdf36844759880012bef0dde2c, entries=150, sequenceid=288, filesize=30.5 K 2024-11-19T17:50:53,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/eddf5f0da8564a9b81bbe9f25e3972fb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/eddf5f0da8564a9b81bbe9f25e3972fb 2024-11-19T17:50:53,737 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/eddf5f0da8564a9b81bbe9f25e3972fb, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T17:50:53,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/522b0fd5064c4eff91fa56e2b982ed3d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/522b0fd5064c4eff91fa56e2b982ed3d 2024-11-19T17:50:53,743 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/522b0fd5064c4eff91fa56e2b982ed3d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T17:50:53,744 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for b9c37934b4aa3330f167e8a793f13680 in 915ms, sequenceid=288, compaction requested=false 2024-11-19T17:50:53,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:53,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:53,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-19T17:50:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-19T17:50:53,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-19T17:50:53,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3750 sec 2024-11-19T17:50:53,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.3800 sec 2024-11-19T17:50:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:54,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:54,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-19T17:50:54,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192002dfd760484753a88f2da4ae0f3bb8_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038654119/Put/seqid=0 2024-11-19T17:50:54,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742455_1631 (size=14994) 2024-11-19T17:50:54,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038714133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038714135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41006 deadline: 1732038714149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,149 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_0' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41100 deadline: 1732038714159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,161 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26d3106abcfe,36839,1732038481522 for 'test_row_1' in b9c37934b4aa3330f167e8a793f13680 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T17:50:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038714198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038714249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038714249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038714459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038714466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:50:54,518 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:54,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-19T17:50:54,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:50:54,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:54,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:54,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,539 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192002dfd760484753a88f2da4ae0f3bb8_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192002dfd760484753a88f2da4ae0f3bb8_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:54,539 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/643aaf003be84110a632252376aa8bda, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:54,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/643aaf003be84110a632252376aa8bda is 175, key is test_row_0/A:col10/1732038654119/Put/seqid=0 2024-11-19T17:50:54,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742456_1632 (size=39949) 2024-11-19T17:50:54,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/643aaf003be84110a632252376aa8bda 2024-11-19T17:50:54,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/332625d7d578432cbd916e08b86c6fae is 50, key is test_row_0/B:col10/1732038654119/Put/seqid=0 2024-11-19T17:50:54,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742457_1633 (size=12301) 2024-11-19T17:50:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:50:54,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T17:50:54,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:54,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038714768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038714778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:54,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T17:50:54,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:54,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:50:54,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/332625d7d578432cbd916e08b86c6fae 2024-11-19T17:50:54,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/287cb4893f3c40018e0794a2a826efcb is 50, key is test_row_0/C:col10/1732038654119/Put/seqid=0 2024-11-19T17:50:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742458_1634 (size=12301) 2024-11-19T17:50:54,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/287cb4893f3c40018e0794a2a826efcb 2024-11-19T17:50:54,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/643aaf003be84110a632252376aa8bda as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/643aaf003be84110a632252376aa8bda 2024-11-19T17:50:54,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/643aaf003be84110a632252376aa8bda, entries=200, sequenceid=319, filesize=39.0 K 2024-11-19T17:50:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/332625d7d578432cbd916e08b86c6fae as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/332625d7d578432cbd916e08b86c6fae 2024-11-19T17:50:54,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T17:50:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:54,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/332625d7d578432cbd916e08b86c6fae, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T17:50:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/287cb4893f3c40018e0794a2a826efcb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/287cb4893f3c40018e0794a2a826efcb 2024-11-19T17:50:54,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/287cb4893f3c40018e0794a2a826efcb, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T17:50:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for b9c37934b4aa3330f167e8a793f13680 in 866ms, sequenceid=319, compaction requested=true 2024-11-19T17:50:54,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:54,986 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:54,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:54,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:54,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:54,986 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:54,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:54,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,987 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:54,987 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:50:54,987 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,987 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,987 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:50:54,987 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/90394742eaa54a95b8c33ae7b490039b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/19880fcdf36844759880012bef0dde2c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/643aaf003be84110a632252376aa8bda] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=100.7 K 2024-11-19T17:50:54,987 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,987 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:54,987 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/bc8febbeac2d4bf09c19f6f0b4a9e46f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/eddf5f0da8564a9b81bbe9f25e3972fb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/332625d7d578432cbd916e08b86c6fae] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=36.7 K 2024-11-19T17:50:54,987 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/90394742eaa54a95b8c33ae7b490039b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/19880fcdf36844759880012bef0dde2c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/643aaf003be84110a632252376aa8bda] 2024-11-19T17:50:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,988 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90394742eaa54a95b8c33ae7b490039b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732038650595 2024-11-19T17:50:54,988 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting bc8febbeac2d4bf09c19f6f0b4a9e46f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732038650595 2024-11-19T17:50:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,989 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19880fcdf36844759880012bef0dde2c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038651773 2024-11-19T17:50:54,989 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting eddf5f0da8564a9b81bbe9f25e3972fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038651773 2024-11-19T17:50:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,989 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 332625d7d578432cbd916e08b86c6fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732038653051 2024-11-19T17:50:54,989 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 643aaf003be84110a632252376aa8bda, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732038653051 2024-11-19T17:50:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,001 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,002 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#534 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,002 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/c3d0cffdf4bd40a383806ed696d82505 is 50, key is test_row_0/B:col10/1732038654119/Put/seqid=0 2024-11-19T17:50:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,003 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b28b09719a0943789b1198f28c9e04f0_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,004 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b28b09719a0943789b1198f28c9e04f0_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:55,005 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b28b09719a0943789b1198f28c9e04f0_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742459_1635 (size=13051) 2024-11-19T17:50:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742460_1636 (size=4469) 2024-11-19T17:50:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T17:50:55,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,131 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T17:50:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:50:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119899b83e9ec874d018704165ac039b350_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_1/A:col10/1732038654132/Put/seqid=0 2024-11-19T17:50:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742461_1637 (size=9914) 2024-11-19T17:50:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,174 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119899b83e9ec874d018704165ac039b350_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119899b83e9ec874d018704165ac039b350_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/ce79e0c01d70411fabcc7b28625b8708, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:55,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/ce79e0c01d70411fabcc7b28625b8708 is 175, key is test_row_1/A:col10/1732038654132/Put/seqid=0 2024-11-19T17:50:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742462_1638 (size=22561) 2024-11-19T17:50:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:55,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:55,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038715302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038715302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038715303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038715419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038715419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038715419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,423 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/c3d0cffdf4bd40a383806ed696d82505 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c3d0cffdf4bd40a383806ed696d82505 2024-11-19T17:50:55,427 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into c3d0cffdf4bd40a383806ed696d82505(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:55,427 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:55,428 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=13, startTime=1732038654986; duration=0sec 2024-11-19T17:50:55,428 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:55,428 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:50:55,428 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:55,428 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:55,428 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:50:55,428 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:55,428 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/c2b7913b9fc54210aff16e513c0cbf69, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/522b0fd5064c4eff91fa56e2b982ed3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/287cb4893f3c40018e0794a2a826efcb] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=36.7 K 2024-11-19T17:50:55,429 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b7913b9fc54210aff16e513c0cbf69, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732038650595 2024-11-19T17:50:55,429 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 522b0fd5064c4eff91fa56e2b982ed3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732038651773 2024-11-19T17:50:55,429 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 287cb4893f3c40018e0794a2a826efcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732038653051 2024-11-19T17:50:55,434 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:55,434 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/0f62c8f64a224ed58d9da2c7ac3f7ad7 is 50, key is test_row_0/C:col10/1732038654119/Put/seqid=0 2024-11-19T17:50:55,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742463_1639 (size=13051) 2024-11-19T17:50:55,443 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#535 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:55,443 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/610d375a37a843fdb31abe163938f61e is 175, key is test_row_0/A:col10/1732038654119/Put/seqid=0 2024-11-19T17:50:55,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742464_1640 (size=32005) 2024-11-19T17:50:55,602 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/ce79e0c01d70411fabcc7b28625b8708 2024-11-19T17:50:55,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/7ed8e27d59444948a82fc56d3972f826 is 50, key is test_row_1/B:col10/1732038654132/Put/seqid=0 2024-11-19T17:50:55,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742465_1641 (size=9857) 2024-11-19T17:50:55,615 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/7ed8e27d59444948a82fc56d3972f826 2024-11-19T17:50:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/12309489aa6b476f8e99faae528d798e is 50, key is test_row_1/C:col10/1732038654132/Put/seqid=0 2024-11-19T17:50:55,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742466_1642 (size=9857) 2024-11-19T17:50:55,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038715628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038715628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038715628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:50:55,840 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/0f62c8f64a224ed58d9da2c7ac3f7ad7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0f62c8f64a224ed58d9da2c7ac3f7ad7 2024-11-19T17:50:55,844 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into 0f62c8f64a224ed58d9da2c7ac3f7ad7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:55,844 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:55,844 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=13, startTime=1732038654986; duration=0sec 2024-11-19T17:50:55,844 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:55,844 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:50:55,852 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/610d375a37a843fdb31abe163938f61e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/610d375a37a843fdb31abe163938f61e 2024-11-19T17:50:55,857 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into 610d375a37a843fdb31abe163938f61e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:55,857 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:55,857 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=13, startTime=1732038654986; duration=0sec 2024-11-19T17:50:55,857 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:55,857 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:50:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038715939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038715939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038715939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,025 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/12309489aa6b476f8e99faae528d798e 2024-11-19T17:50:56,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/ce79e0c01d70411fabcc7b28625b8708 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce79e0c01d70411fabcc7b28625b8708 2024-11-19T17:50:56,031 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce79e0c01d70411fabcc7b28625b8708, entries=100, sequenceid=326, filesize=22.0 K 2024-11-19T17:50:56,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/7ed8e27d59444948a82fc56d3972f826 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/7ed8e27d59444948a82fc56d3972f826 2024-11-19T17:50:56,036 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/7ed8e27d59444948a82fc56d3972f826, entries=100, sequenceid=326, filesize=9.6 K 2024-11-19T17:50:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/12309489aa6b476f8e99faae528d798e as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/12309489aa6b476f8e99faae528d798e 2024-11-19T17:50:56,041 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/12309489aa6b476f8e99faae528d798e, entries=100, sequenceid=326, filesize=9.6 K 2024-11-19T17:50:56,042 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for b9c37934b4aa3330f167e8a793f13680 in 911ms, sequenceid=326, compaction requested=false 2024-11-19T17:50:56,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:56,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:56,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-19T17:50:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-19T17:50:56,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-19T17:50:56,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5220 sec 2024-11-19T17:50:56,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.5280 sec 2024-11-19T17:50:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:56,450 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:56,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-19T17:50:56,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038716452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038716452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038716456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d4b6fa9601a343ccb203d2f0d77598a7_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038655301/Put/seqid=0 2024-11-19T17:50:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742467_1643 (size=12454) 2024-11-19T17:50:56,470 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,473 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d4b6fa9601a343ccb203d2f0d77598a7_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d4b6fa9601a343ccb203d2f0d77598a7_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:56,474 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/4f7c3378283940ca8b2eff0f55a82d1d, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:56,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/4f7c3378283940ca8b2eff0f55a82d1d is 175, key is test_row_0/A:col10/1732038655301/Put/seqid=0 2024-11-19T17:50:56,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742468_1644 (size=31255) 2024-11-19T17:50:56,478 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/4f7c3378283940ca8b2eff0f55a82d1d 2024-11-19T17:50:56,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/0b909fb3266442c0a6a25f5839dbe0d5 is 50, key is test_row_0/B:col10/1732038655301/Put/seqid=0 2024-11-19T17:50:56,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742469_1645 (size=12301) 2024-11-19T17:50:56,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038716559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038716569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038716569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:50:56,668 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:56,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:50:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-19T17:50:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:50:56,670 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:50:56,671 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:50:56,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:50:56,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038716768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:50:56,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038716779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038716779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:56,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T17:50:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:56,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:56,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/0b909fb3266442c0a6a25f5839dbe0d5 2024-11-19T17:50:56,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/fecf06d871034cd6992ebece031b54f8 is 50, key is test_row_0/C:col10/1732038655301/Put/seqid=0 2024-11-19T17:50:56,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742470_1646 (size=12301) 2024-11-19T17:50:56,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/fecf06d871034cd6992ebece031b54f8 2024-11-19T17:50:56,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/4f7c3378283940ca8b2eff0f55a82d1d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/4f7c3378283940ca8b2eff0f55a82d1d 2024-11-19T17:50:56,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/4f7c3378283940ca8b2eff0f55a82d1d, entries=150, sequenceid=359, filesize=30.5 K 2024-11-19T17:50:56,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/0b909fb3266442c0a6a25f5839dbe0d5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0b909fb3266442c0a6a25f5839dbe0d5 2024-11-19T17:50:56,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0b909fb3266442c0a6a25f5839dbe0d5, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T17:50:56,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/fecf06d871034cd6992ebece031b54f8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/fecf06d871034cd6992ebece031b54f8 2024-11-19T17:50:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/fecf06d871034cd6992ebece031b54f8, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T17:50:56,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for b9c37934b4aa3330f167e8a793f13680 in 472ms, sequenceid=359, compaction requested=true 2024-11-19T17:50:56,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:56,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:50:56,922 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:56,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:56,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:50:56,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:56,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:50:56,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:50:56,922 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,924 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:56,924 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:50:56,924 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,924 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/610d375a37a843fdb31abe163938f61e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce79e0c01d70411fabcc7b28625b8708, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/4f7c3378283940ca8b2eff0f55a82d1d] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=83.8 K 2024-11-19T17:50:56,924 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:56,924 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/610d375a37a843fdb31abe163938f61e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce79e0c01d70411fabcc7b28625b8708, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/4f7c3378283940ca8b2eff0f55a82d1d] 2024-11-19T17:50:56,924 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,924 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:50:56,924 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:56,924 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 610d375a37a843fdb31abe163938f61e, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732038653051 2024-11-19T17:50:56,925 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c3d0cffdf4bd40a383806ed696d82505, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/7ed8e27d59444948a82fc56d3972f826, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0b909fb3266442c0a6a25f5839dbe0d5] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=34.4 K 2024-11-19T17:50:56,925 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce79e0c01d70411fabcc7b28625b8708, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732038654131 2024-11-19T17:50:56,925 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting c3d0cffdf4bd40a383806ed696d82505, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732038653051 2024-11-19T17:50:56,925 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed8e27d59444948a82fc56d3972f826, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732038654131 2024-11-19T17:50:56,925 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f7c3378283940ca8b2eff0f55a82d1d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732038655301 2024-11-19T17:50:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,925 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b909fb3266442c0a6a25f5839dbe0d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732038655301 2024-11-19T17:50:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,950 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#543 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:56,950 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/0a6b4f1f2d17493ba3a886915940986d is 50, key is test_row_0/B:col10/1732038655301/Put/seqid=0 2024-11-19T17:50:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,951 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:56,954 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119f0ad5f6994634dfe9f88b9dc092f9589_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,956 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119f0ad5f6994634dfe9f88b9dc092f9589_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:56,956 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f0ad5f6994634dfe9f88b9dc092f9589_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742472_1648 (size=4469) 2024-11-19T17:50:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,976 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#544 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36839 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T17:50:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:56,977 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-19T17:50:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742471_1647 (size=13153) 2024-11-19T17:50:56,977 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/76ddf3f6bbcf4032ba6cc73737b24080 is 175, key is test_row_0/A:col10/1732038655301/Put/seqid=0 2024-11-19T17:50:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:50:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742473_1649 (size=32107) 2024-11-19T17:50:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119238c02e3fd9141f885a5e310a9edca33_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038656450/Put/seqid=0 2024-11-19T17:50:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742474_1650 (size=9914) 2024-11-19T17:50:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:57,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. as already flushing 2024-11-19T17:50:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038717145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038717145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038717150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038717259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038717259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038717259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:50:57,381 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/0a6b4f1f2d17493ba3a886915940986d as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0a6b4f1f2d17493ba3a886915940986d 2024-11-19T17:50:57,384 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into 0a6b4f1f2d17493ba3a886915940986d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:57,384 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:57,384 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=13, startTime=1732038656922; duration=0sec 2024-11-19T17:50:57,385 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:50:57,385 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:50:57,385 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:50:57,386 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:50:57,386 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:50:57,386 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:57,386 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0f62c8f64a224ed58d9da2c7ac3f7ad7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/12309489aa6b476f8e99faae528d798e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/fecf06d871034cd6992ebece031b54f8] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=34.4 K 2024-11-19T17:50:57,387 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f62c8f64a224ed58d9da2c7ac3f7ad7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732038653051 2024-11-19T17:50:57,387 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 12309489aa6b476f8e99faae528d798e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732038654131 2024-11-19T17:50:57,387 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting fecf06d871034cd6992ebece031b54f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732038655301 2024-11-19T17:50:57,393 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#546 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:50:57,393 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/1b8179ade9d1425090d0b94cc52ae2a7 is 50, key is test_row_0/C:col10/1732038655301/Put/seqid=0 2024-11-19T17:50:57,401 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/76ddf3f6bbcf4032ba6cc73737b24080 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/76ddf3f6bbcf4032ba6cc73737b24080 2024-11-19T17:50:57,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742475_1651 (size=13153) 2024-11-19T17:50:57,406 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/1b8179ade9d1425090d0b94cc52ae2a7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1b8179ade9d1425090d0b94cc52ae2a7 2024-11-19T17:50:57,406 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into 76ddf3f6bbcf4032ba6cc73737b24080(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:57,406 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:57,406 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=13, startTime=1732038656922; duration=0sec 2024-11-19T17:50:57,406 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:57,406 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:50:57,410 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into 1b8179ade9d1425090d0b94cc52ae2a7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:50:57,410 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:57,410 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=13, startTime=1732038656922; duration=0sec 2024-11-19T17:50:57,410 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:50:57,410 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:50:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:57,441 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119238c02e3fd9141f885a5e310a9edca33_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119238c02e3fd9141f885a5e310a9edca33_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:57,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/240b3501d73e426696ed23b61c9c365a, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:57,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/240b3501d73e426696ed23b61c9c365a is 175, key is test_row_0/A:col10/1732038656450/Put/seqid=0 2024-11-19T17:50:57,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742476_1652 (size=22561) 2024-11-19T17:50:57,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038717468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038717468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038717469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,577 INFO [Thread-2444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2444. 2024-11-19T17:50:57,577 DEBUG [Thread-2444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:57,577 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,577 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,577 INFO [Thread-2446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2446. 2024-11-19T17:50:57,577 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:57,577 DEBUG [Thread-2446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:57,577 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,578 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:57,579 INFO [Thread-2442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2442. 2024-11-19T17:50:57,579 DEBUG [Thread-2442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:57,579 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,579 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:57,580 INFO [Thread-2440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2440. 2024-11-19T17:50:57,580 DEBUG [Thread-2440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:57,580 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,581 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:57,581 INFO [Thread-2448 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2448. 2024-11-19T17:50:57,581 DEBUG [Thread-2448 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:57,581 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,581 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:57,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038717779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038717779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038717779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:50:57,851 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/240b3501d73e426696ed23b61c9c365a 2024-11-19T17:50:57,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/5ed5756f0a014c03bc339b229e7cbbd7 is 50, key is test_row_0/B:col10/1732038656450/Put/seqid=0 2024-11-19T17:50:57,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742477_1653 (size=9857) 2024-11-19T17:50:58,261 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/5ed5756f0a014c03bc339b229e7cbbd7 2024-11-19T17:50:58,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/d1ba56fc3d9642e2adf49da91bd447fd is 50, key is test_row_0/C:col10/1732038656450/Put/seqid=0 2024-11-19T17:50:58,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742478_1654 (size=9857) 2024-11-19T17:50:58,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:58,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41090 deadline: 1732038718288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41080 deadline: 1732038718288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:58,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T17:50:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41048 deadline: 1732038718288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 , the old value is region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:50:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., hostname=26d3106abcfe,36839,1732038481522, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:50:58,671 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/d1ba56fc3d9642e2adf49da91bd447fd 2024-11-19T17:50:58,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/240b3501d73e426696ed23b61c9c365a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/240b3501d73e426696ed23b61c9c365a 2024-11-19T17:50:58,679 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/240b3501d73e426696ed23b61c9c365a, entries=100, sequenceid=365, filesize=22.0 K 2024-11-19T17:50:58,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/5ed5756f0a014c03bc339b229e7cbbd7 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/5ed5756f0a014c03bc339b229e7cbbd7 2024-11-19T17:50:58,683 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/5ed5756f0a014c03bc339b229e7cbbd7, entries=100, sequenceid=365, filesize=9.6 K 2024-11-19T17:50:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/d1ba56fc3d9642e2adf49da91bd447fd as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d1ba56fc3d9642e2adf49da91bd447fd 2024-11-19T17:50:58,687 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d1ba56fc3d9642e2adf49da91bd447fd, entries=100, sequenceid=365, filesize=9.6 K 2024-11-19T17:50:58,688 INFO [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for b9c37934b4aa3330f167e8a793f13680 in 1711ms, sequenceid=365, compaction requested=false 2024-11-19T17:50:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:50:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:50:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-19T17:50:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-19T17:50:58,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-19T17:50:58,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0180 sec 2024-11-19T17:50:58,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.0230 sec 2024-11-19T17:50:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:50:58,819 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:50:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36839 {}] regionserver.HRegion(8855): Flush requested on b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T17:50:59,302 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-19T17:50:59,302 INFO [Thread-2435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2435. 2024-11-19T17:50:59,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-19T17:50:59,302 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:59,302 DEBUG [Thread-2435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:59,302 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:59,302 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:59,302 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:59,302 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:59,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:59,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:59,302 INFO [Thread-2437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2437. 2024-11-19T17:50:59,302 DEBUG [Thread-2437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:50:59,302 DEBUG [Thread-2437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:59,303 DEBUG [Thread-2437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:50:59,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:50:59,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198527af70fbf54901bba8b7ba2bc18ef7_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038657142/Put/seqid=0 2024-11-19T17:50:59,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742479_1655 (size=12454) 2024-11-19T17:50:59,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:59,319 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198527af70fbf54901bba8b7ba2bc18ef7_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198527af70fbf54901bba8b7ba2bc18ef7_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:50:59,320 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/5b1259c89b944d19b9993157ba0425d8, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:50:59,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/5b1259c89b944d19b9993157ba0425d8 is 175, key is test_row_0/A:col10/1732038657142/Put/seqid=0 2024-11-19T17:50:59,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742480_1656 (size=31255) 2024-11-19T17:50:59,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:50:59,725 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/5b1259c89b944d19b9993157ba0425d8 2024-11-19T17:50:59,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/201e7c99eff5437ebadfe0a1cd5a09ff is 50, key is test_row_0/B:col10/1732038657142/Put/seqid=0 2024-11-19T17:50:59,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742481_1657 (size=12301) 2024-11-19T17:51:00,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/201e7c99eff5437ebadfe0a1cd5a09ff 2024-11-19T17:51:00,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/d7e369245b7f40b8a01a7d6c1a4184b2 is 50, key is test_row_0/C:col10/1732038657142/Put/seqid=0 2024-11-19T17:51:00,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742482_1658 (size=12301) 2024-11-19T17:51:00,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/d7e369245b7f40b8a01a7d6c1a4184b2 2024-11-19T17:51:00,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/5b1259c89b944d19b9993157ba0425d8 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/5b1259c89b944d19b9993157ba0425d8 2024-11-19T17:51:00,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/5b1259c89b944d19b9993157ba0425d8, entries=150, sequenceid=401, filesize=30.5 K 2024-11-19T17:51:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/201e7c99eff5437ebadfe0a1cd5a09ff as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/201e7c99eff5437ebadfe0a1cd5a09ff 2024-11-19T17:51:00,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/201e7c99eff5437ebadfe0a1cd5a09ff, entries=150, sequenceid=401, filesize=12.0 K 2024-11-19T17:51:00,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/d7e369245b7f40b8a01a7d6c1a4184b2 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d7e369245b7f40b8a01a7d6c1a4184b2 2024-11-19T17:51:00,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d7e369245b7f40b8a01a7d6c1a4184b2, entries=150, sequenceid=401, filesize=12.0 K 2024-11-19T17:51:00,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for b9c37934b4aa3330f167e8a793f13680 in 859ms, sequenceid=401, compaction requested=true 2024-11-19T17:51:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:51:00,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:51:00,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:51:00,162 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:51:00,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:51:00,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:51:00,162 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:51:00,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9c37934b4aa3330f167e8a793f13680:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:51:00,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:51:00,162 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:51:00,162 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/A is initiating minor compaction (all files) 2024-11-19T17:51:00,162 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:51:00,163 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/A in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:51:00,163 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/B is initiating minor compaction (all files) 2024-11-19T17:51:00,163 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/B in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:51:00,163 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/76ddf3f6bbcf4032ba6cc73737b24080, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/240b3501d73e426696ed23b61c9c365a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/5b1259c89b944d19b9993157ba0425d8] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=83.9 K 2024-11-19T17:51:00,163 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:51:00,163 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0a6b4f1f2d17493ba3a886915940986d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/5ed5756f0a014c03bc339b229e7cbbd7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/201e7c99eff5437ebadfe0a1cd5a09ff] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=34.5 K 2024-11-19T17:51:00,163 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. files: [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/76ddf3f6bbcf4032ba6cc73737b24080, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/240b3501d73e426696ed23b61c9c365a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/5b1259c89b944d19b9993157ba0425d8] 2024-11-19T17:51:00,163 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76ddf3f6bbcf4032ba6cc73737b24080, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732038655301 2024-11-19T17:51:00,163 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6b4f1f2d17493ba3a886915940986d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732038655301 2024-11-19T17:51:00,163 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240b3501d73e426696ed23b61c9c365a, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732038656450 2024-11-19T17:51:00,163 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed5756f0a014c03bc339b229e7cbbd7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732038656450 2024-11-19T17:51:00,163 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b1259c89b944d19b9993157ba0425d8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732038657142 2024-11-19T17:51:00,163 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 201e7c99eff5437ebadfe0a1cd5a09ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732038657142 2024-11-19T17:51:00,169 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:51:00,170 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#B#compaction#552 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:51:00,170 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411190c9100a393754b48b61aa6296f58e9f1_b9c37934b4aa3330f167e8a793f13680 store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:51:00,171 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/04f7ae224830441fa7b23d80aebbd577 is 50, key is test_row_0/B:col10/1732038657142/Put/seqid=0 2024-11-19T17:51:00,173 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411190c9100a393754b48b61aa6296f58e9f1_b9c37934b4aa3330f167e8a793f13680, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:51:00,174 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190c9100a393754b48b61aa6296f58e9f1_b9c37934b4aa3330f167e8a793f13680 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:51:00,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742483_1659 (size=13255) 2024-11-19T17:51:00,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742484_1660 (size=4469) 2024-11-19T17:51:00,178 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/04f7ae224830441fa7b23d80aebbd577 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/04f7ae224830441fa7b23d80aebbd577 2024-11-19T17:51:00,182 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/B of b9c37934b4aa3330f167e8a793f13680 into 04f7ae224830441fa7b23d80aebbd577(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:51:00,182 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:51:00,182 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/B, priority=13, startTime=1732038660162; duration=0sec 2024-11-19T17:51:00,182 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:51:00,182 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:B 2024-11-19T17:51:00,182 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:51:00,183 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:51:00,183 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1541): b9c37934b4aa3330f167e8a793f13680/C is initiating minor compaction (all files) 2024-11-19T17:51:00,183 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9c37934b4aa3330f167e8a793f13680/C in TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:51:00,183 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1b8179ade9d1425090d0b94cc52ae2a7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d1ba56fc3d9642e2adf49da91bd447fd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d7e369245b7f40b8a01a7d6c1a4184b2] into tmpdir=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp, totalSize=34.5 K 2024-11-19T17:51:00,183 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8179ade9d1425090d0b94cc52ae2a7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732038655301 2024-11-19T17:51:00,184 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ba56fc3d9642e2adf49da91bd447fd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732038656450 2024-11-19T17:51:00,184 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e369245b7f40b8a01a7d6c1a4184b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732038657142 2024-11-19T17:51:00,190 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#C#compaction#554 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:51:00,191 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/c6bab54ede8743198e2d33b52a40e1ae is 50, key is test_row_0/C:col10/1732038657142/Put/seqid=0 2024-11-19T17:51:00,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742485_1661 (size=13255) 2024-11-19T17:51:00,578 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9c37934b4aa3330f167e8a793f13680#A#compaction#553 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T17:51:00,579 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/2907c1f36a22496681e5bb17806a62e5 is 175, key is test_row_0/A:col10/1732038657142/Put/seqid=0 2024-11-19T17:51:00,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742486_1662 (size=32209) 2024-11-19T17:51:00,600 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/c6bab54ede8743198e2d33b52a40e1ae as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/c6bab54ede8743198e2d33b52a40e1ae 2024-11-19T17:51:00,606 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/C of b9c37934b4aa3330f167e8a793f13680 into c6bab54ede8743198e2d33b52a40e1ae(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:51:00,606 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:51:00,606 INFO [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/C, priority=13, startTime=1732038660162; duration=0sec 2024-11-19T17:51:00,606 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:51:00,606 DEBUG [RS:0;26d3106abcfe:36839-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:C 2024-11-19T17:51:01,009 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/2907c1f36a22496681e5bb17806a62e5 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2907c1f36a22496681e5bb17806a62e5 2024-11-19T17:51:01,014 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9c37934b4aa3330f167e8a793f13680/A of b9c37934b4aa3330f167e8a793f13680 into 2907c1f36a22496681e5bb17806a62e5(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:51:01,014 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9c37934b4aa3330f167e8a793f13680: 2024-11-19T17:51:01,014 INFO [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., storeName=b9c37934b4aa3330f167e8a793f13680/A, priority=13, startTime=1732038660162; duration=0sec 2024-11-19T17:51:01,015 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:51:01,015 DEBUG [RS:0;26d3106abcfe:36839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9c37934b4aa3330f167e8a793f13680:A 2024-11-19T17:51:04,208 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-19T17:51:04,208 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:51:04,208 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:04,208 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:04,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:51:04,239 INFO [Thread-2433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2433. 2024-11-19T17:51:04,239 DEBUG [Thread-2433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:51:04,239 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:04,240 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:04,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 164 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3486 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3506 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3400 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3463 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3495 2024-11-19T17:51:04,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:51:04,240 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:51:04,240 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:51:04,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:04,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:04,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:51:04,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:51:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:51:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:51:04,243 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038664243"}]},"ts":"1732038664243"} 2024-11-19T17:51:04,245 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:51:04,245 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:51:04,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:51:04,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, UNASSIGN}] 2024-11-19T17:51:04,247 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, UNASSIGN 2024-11-19T17:51:04,248 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=b9c37934b4aa3330f167e8a793f13680, regionState=CLOSING, regionLocation=26d3106abcfe,36839,1732038481522 2024-11-19T17:51:04,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, UNASSIGN because future has completed 2024-11-19T17:51:04,249 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:51:04,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522}] 2024-11-19T17:51:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:51:04,401 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:04,401 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:51:04,401 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing b9c37934b4aa3330f167e8a793f13680, disabling compactions & flushes 2024-11-19T17:51:04,401 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:51:04,401 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:51:04,401 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. after waiting 0 ms 2024-11-19T17:51:04,401 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:51:04,402 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing b9c37934b4aa3330f167e8a793f13680 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-19T17:51:04,407 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a6f97cdbdd3040b093a1df210e23180b_b9c37934b4aa3330f167e8a793f13680 is 50, key is test_row_0/A:col10/1732038664204/Put/seqid=0 2024-11-19T17:51:04,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742487_1663 (size=9914) 2024-11-19T17:51:04,477 DEBUG [regionserver/26d3106abcfe:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-19T17:51:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:51:04,811 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:51:04,814 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a6f97cdbdd3040b093a1df210e23180b_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a6f97cdbdd3040b093a1df210e23180b_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:04,815 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b5b1e652408b46c68b0dd5421793f8ec, store: [table=TestAcidGuarantees family=A region=b9c37934b4aa3330f167e8a793f13680] 2024-11-19T17:51:04,816 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b5b1e652408b46c68b0dd5421793f8ec is 175, key is test_row_0/A:col10/1732038664204/Put/seqid=0 2024-11-19T17:51:04,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742488_1664 (size=22561) 2024-11-19T17:51:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:51:05,219 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b5b1e652408b46c68b0dd5421793f8ec 2024-11-19T17:51:05,223 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/c8d46161b7bc487487a2efbda75ee10c is 50, key is test_row_0/B:col10/1732038664204/Put/seqid=0 2024-11-19T17:51:05,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742489_1665 (size=9857) 2024-11-19T17:51:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:51:05,627 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/c8d46161b7bc487487a2efbda75ee10c 2024-11-19T17:51:05,637 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/57df53b7040e4b56b224003e7688d647 is 50, key is test_row_0/C:col10/1732038664204/Put/seqid=0 2024-11-19T17:51:05,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742490_1666 (size=9857) 2024-11-19T17:51:06,047 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/57df53b7040e4b56b224003e7688d647 2024-11-19T17:51:06,050 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/A/b5b1e652408b46c68b0dd5421793f8ec as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b5b1e652408b46c68b0dd5421793f8ec 2024-11-19T17:51:06,053 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b5b1e652408b46c68b0dd5421793f8ec, entries=100, sequenceid=409, filesize=22.0 K 2024-11-19T17:51:06,054 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/B/c8d46161b7bc487487a2efbda75ee10c as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c8d46161b7bc487487a2efbda75ee10c 2024-11-19T17:51:06,056 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c8d46161b7bc487487a2efbda75ee10c, entries=100, sequenceid=409, filesize=9.6 K 2024-11-19T17:51:06,057 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/.tmp/C/57df53b7040e4b56b224003e7688d647 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/57df53b7040e4b56b224003e7688d647 2024-11-19T17:51:06,059 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/57df53b7040e4b56b224003e7688d647, entries=100, sequenceid=409, filesize=9.6 K 2024-11-19T17:51:06,060 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for b9c37934b4aa3330f167e8a793f13680 in 1659ms, sequenceid=409, compaction requested=false 2024-11-19T17:51:06,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/231407d5fef34092afb6d4cdefaf2350, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce007b4810f84836aa41832a7085930a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b214be388bd64b00b06e6e74f300cd6f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/7c82b62b35c546cd83f2bbdc5b88b48b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/dfd1e7f607d04abd9f8a74c12c7938ef, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/196f32edaa00460b9ecd1a3d4e9541c8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/3ab3f747b598411aa0e3f2385e3664c5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/11ce7f8965ec4500bdafe27cacbbec8b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/8d07b48006c6429fbc996a8da126e8c2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/324523b87c354f6083754ed351f5cc9c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/a93bb4d071c54ebb800ad8821ce6a0b3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/23bc0aae60d34d5d8ed4945575946502, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/55ded3ec8ea84c709994d783dfea208f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/631022858eb84a9da1bf9be2e9d4c982, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b859191799464161bad0460f9ba1b578, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/1d9f38bfc3704b6f9053a73bcb55523e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2ab7813d7a2241928859330795667739, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/cb0240f4f66c4e519348a3e3400fd161, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/9cb157caaf754b65885f8c8cc8dd9359, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/90394742eaa54a95b8c33ae7b490039b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/19880fcdf36844759880012bef0dde2c, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/643aaf003be84110a632252376aa8bda, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/610d375a37a843fdb31abe163938f61e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce79e0c01d70411fabcc7b28625b8708, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/76ddf3f6bbcf4032ba6cc73737b24080, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/4f7c3378283940ca8b2eff0f55a82d1d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/240b3501d73e426696ed23b61c9c365a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/5b1259c89b944d19b9993157ba0425d8] to archive 2024-11-19T17:51:06,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:51:06,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/231407d5fef34092afb6d4cdefaf2350 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/231407d5fef34092afb6d4cdefaf2350 2024-11-19T17:51:06,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce007b4810f84836aa41832a7085930a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce007b4810f84836aa41832a7085930a 2024-11-19T17:51:06,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b214be388bd64b00b06e6e74f300cd6f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b214be388bd64b00b06e6e74f300cd6f 2024-11-19T17:51:06,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/7c82b62b35c546cd83f2bbdc5b88b48b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/7c82b62b35c546cd83f2bbdc5b88b48b 2024-11-19T17:51:06,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/dfd1e7f607d04abd9f8a74c12c7938ef to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/dfd1e7f607d04abd9f8a74c12c7938ef 2024-11-19T17:51:06,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/196f32edaa00460b9ecd1a3d4e9541c8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/196f32edaa00460b9ecd1a3d4e9541c8 2024-11-19T17:51:06,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/3ab3f747b598411aa0e3f2385e3664c5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/3ab3f747b598411aa0e3f2385e3664c5 2024-11-19T17:51:06,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/11ce7f8965ec4500bdafe27cacbbec8b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/11ce7f8965ec4500bdafe27cacbbec8b 2024-11-19T17:51:06,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/8d07b48006c6429fbc996a8da126e8c2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/8d07b48006c6429fbc996a8da126e8c2 2024-11-19T17:51:06,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/324523b87c354f6083754ed351f5cc9c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/324523b87c354f6083754ed351f5cc9c 2024-11-19T17:51:06,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/a93bb4d071c54ebb800ad8821ce6a0b3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/a93bb4d071c54ebb800ad8821ce6a0b3 2024-11-19T17:51:06,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/23bc0aae60d34d5d8ed4945575946502 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/23bc0aae60d34d5d8ed4945575946502 2024-11-19T17:51:06,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/55ded3ec8ea84c709994d783dfea208f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/55ded3ec8ea84c709994d783dfea208f 2024-11-19T17:51:06,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/631022858eb84a9da1bf9be2e9d4c982 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/631022858eb84a9da1bf9be2e9d4c982 2024-11-19T17:51:06,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b859191799464161bad0460f9ba1b578 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b859191799464161bad0460f9ba1b578 2024-11-19T17:51:06,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/1d9f38bfc3704b6f9053a73bcb55523e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/1d9f38bfc3704b6f9053a73bcb55523e 2024-11-19T17:51:06,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2ab7813d7a2241928859330795667739 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2ab7813d7a2241928859330795667739 2024-11-19T17:51:06,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/cb0240f4f66c4e519348a3e3400fd161 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/cb0240f4f66c4e519348a3e3400fd161 2024-11-19T17:51:06,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/9cb157caaf754b65885f8c8cc8dd9359 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/9cb157caaf754b65885f8c8cc8dd9359 2024-11-19T17:51:06,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/90394742eaa54a95b8c33ae7b490039b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/90394742eaa54a95b8c33ae7b490039b 2024-11-19T17:51:06,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/19880fcdf36844759880012bef0dde2c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/19880fcdf36844759880012bef0dde2c 2024-11-19T17:51:06,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/643aaf003be84110a632252376aa8bda to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/643aaf003be84110a632252376aa8bda 2024-11-19T17:51:06,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/610d375a37a843fdb31abe163938f61e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/610d375a37a843fdb31abe163938f61e 2024-11-19T17:51:06,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce79e0c01d70411fabcc7b28625b8708 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/ce79e0c01d70411fabcc7b28625b8708 2024-11-19T17:51:06,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/76ddf3f6bbcf4032ba6cc73737b24080 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/76ddf3f6bbcf4032ba6cc73737b24080 2024-11-19T17:51:06,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/4f7c3378283940ca8b2eff0f55a82d1d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/4f7c3378283940ca8b2eff0f55a82d1d 2024-11-19T17:51:06,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/240b3501d73e426696ed23b61c9c365a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/240b3501d73e426696ed23b61c9c365a 2024-11-19T17:51:06,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/5b1259c89b944d19b9993157ba0425d8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/5b1259c89b944d19b9993157ba0425d8 2024-11-19T17:51:06,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/24c81d66f175452fa8da48a51b9d3552, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/8e6a63d238ed49b893a4704a0b8b2d3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/80781c5431484c679d84c5ec9aafb33f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/706415366666419c98145e38e7e65b2a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/39b8818c16f44206bd6edcb359772735, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1e4f9f851e72451da7e9598561f9e555, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/3cf871ded25d40599edc1ea64743cd86, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/fb2a7876d296427eb8f6e99ef00d9de5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/6a039118874b41ada5744bc28cf16b19, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/2cd2d97f7a8940289e7fb2d938637927, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/985f4c32e7654a66ba16c98ef606cf38, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/30a101c03c8a4fd5b760b702f3bc44c2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c25252df63a342d0b5639cc106a6cd6b, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/55a4e3fbaab041b291a2ef3a082dc168, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/76db30ee062940c89cdb42f69708b3ff, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/af76a794d0ac4574824839a55a5dba27, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1eea4ff96930495d88b341c637886315, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/342eee0557e2424a945137ffb08f02f2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/bc8febbeac2d4bf09c19f6f0b4a9e46f, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/621e557b54ec45e7bae23ac1c9b98fad, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/eddf5f0da8564a9b81bbe9f25e3972fb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c3d0cffdf4bd40a383806ed696d82505, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/332625d7d578432cbd916e08b86c6fae, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/7ed8e27d59444948a82fc56d3972f826, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0a6b4f1f2d17493ba3a886915940986d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0b909fb3266442c0a6a25f5839dbe0d5, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/5ed5756f0a014c03bc339b229e7cbbd7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/201e7c99eff5437ebadfe0a1cd5a09ff] to archive 2024-11-19T17:51:06,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:51:06,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/24c81d66f175452fa8da48a51b9d3552 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/24c81d66f175452fa8da48a51b9d3552 2024-11-19T17:51:06,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/8e6a63d238ed49b893a4704a0b8b2d3d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/8e6a63d238ed49b893a4704a0b8b2d3d 2024-11-19T17:51:06,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/80781c5431484c679d84c5ec9aafb33f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/80781c5431484c679d84c5ec9aafb33f 2024-11-19T17:51:06,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/706415366666419c98145e38e7e65b2a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/706415366666419c98145e38e7e65b2a 2024-11-19T17:51:06,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/39b8818c16f44206bd6edcb359772735 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/39b8818c16f44206bd6edcb359772735 2024-11-19T17:51:06,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1e4f9f851e72451da7e9598561f9e555 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1e4f9f851e72451da7e9598561f9e555 2024-11-19T17:51:06,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/3cf871ded25d40599edc1ea64743cd86 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/3cf871ded25d40599edc1ea64743cd86 2024-11-19T17:51:06,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/fb2a7876d296427eb8f6e99ef00d9de5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/fb2a7876d296427eb8f6e99ef00d9de5 2024-11-19T17:51:06,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/6a039118874b41ada5744bc28cf16b19 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/6a039118874b41ada5744bc28cf16b19 2024-11-19T17:51:06,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/2cd2d97f7a8940289e7fb2d938637927 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/2cd2d97f7a8940289e7fb2d938637927 2024-11-19T17:51:06,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/985f4c32e7654a66ba16c98ef606cf38 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/985f4c32e7654a66ba16c98ef606cf38 2024-11-19T17:51:06,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/30a101c03c8a4fd5b760b702f3bc44c2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/30a101c03c8a4fd5b760b702f3bc44c2 2024-11-19T17:51:06,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c25252df63a342d0b5639cc106a6cd6b to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c25252df63a342d0b5639cc106a6cd6b 2024-11-19T17:51:06,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/55a4e3fbaab041b291a2ef3a082dc168 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/55a4e3fbaab041b291a2ef3a082dc168 2024-11-19T17:51:06,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/76db30ee062940c89cdb42f69708b3ff to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/76db30ee062940c89cdb42f69708b3ff 2024-11-19T17:51:06,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/af76a794d0ac4574824839a55a5dba27 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/af76a794d0ac4574824839a55a5dba27 2024-11-19T17:51:06,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1eea4ff96930495d88b341c637886315 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/1eea4ff96930495d88b341c637886315 2024-11-19T17:51:06,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/342eee0557e2424a945137ffb08f02f2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/342eee0557e2424a945137ffb08f02f2 2024-11-19T17:51:06,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/bc8febbeac2d4bf09c19f6f0b4a9e46f to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/bc8febbeac2d4bf09c19f6f0b4a9e46f 2024-11-19T17:51:06,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/621e557b54ec45e7bae23ac1c9b98fad to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/621e557b54ec45e7bae23ac1c9b98fad 2024-11-19T17:51:06,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/eddf5f0da8564a9b81bbe9f25e3972fb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/eddf5f0da8564a9b81bbe9f25e3972fb 2024-11-19T17:51:06,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c3d0cffdf4bd40a383806ed696d82505 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c3d0cffdf4bd40a383806ed696d82505 2024-11-19T17:51:06,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/332625d7d578432cbd916e08b86c6fae to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/332625d7d578432cbd916e08b86c6fae 2024-11-19T17:51:06,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/7ed8e27d59444948a82fc56d3972f826 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/7ed8e27d59444948a82fc56d3972f826 2024-11-19T17:51:06,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0a6b4f1f2d17493ba3a886915940986d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0a6b4f1f2d17493ba3a886915940986d 2024-11-19T17:51:06,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0b909fb3266442c0a6a25f5839dbe0d5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/0b909fb3266442c0a6a25f5839dbe0d5 2024-11-19T17:51:06,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/5ed5756f0a014c03bc339b229e7cbbd7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/5ed5756f0a014c03bc339b229e7cbbd7 2024-11-19T17:51:06,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/201e7c99eff5437ebadfe0a1cd5a09ff to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/201e7c99eff5437ebadfe0a1cd5a09ff 2024-11-19T17:51:06,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0a0d706cbbff4338822c91c308baabeb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2cafb924a086462ab6bda3062f5708f4, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/7b5dc31aa7cf48b7930a6ce0b220aee3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/26afb3745abf4a4093e88ab8adac3f02, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f578cc44c69e442ca7077102d5ff1d54, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/94fd63105abf4f17b4f228065bad3a4a, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1efa8376aca347f3aed015795b654076, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/98df6ef2d25740b6aa9637fd9744b2d3, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f92871bcd06d43599e692f4df3b44789, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/dcc5ce986e6a40669d5a9a4bdf9460b2, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3f0e0d06122843649bdf6f36af270964, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/a52fa0f4390b47d08032882e3846b265, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/6a3652b1f6e6489b89e419fcc4944fcd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ae9c016c488e4e0cbe44389aa7129957, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2e27ac4146fd4d24828cb3a0b93cee76, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ff2c254ab23d40ec84a4b65ce3b36b22, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/99482849796b4e6f8bb4241a564dd845, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3151a350fdc444f6aaa4a9eed30486e6, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/c2b7913b9fc54210aff16e513c0cbf69, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/65f6f9918dc047129adb1124376d5b73, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/522b0fd5064c4eff91fa56e2b982ed3d, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0f62c8f64a224ed58d9da2c7ac3f7ad7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/287cb4893f3c40018e0794a2a826efcb, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/12309489aa6b476f8e99faae528d798e, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1b8179ade9d1425090d0b94cc52ae2a7, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/fecf06d871034cd6992ebece031b54f8, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d1ba56fc3d9642e2adf49da91bd447fd, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d7e369245b7f40b8a01a7d6c1a4184b2] to archive 2024-11-19T17:51:06,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:51:06,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0a0d706cbbff4338822c91c308baabeb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0a0d706cbbff4338822c91c308baabeb 2024-11-19T17:51:06,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2cafb924a086462ab6bda3062f5708f4 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2cafb924a086462ab6bda3062f5708f4 2024-11-19T17:51:06,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/7b5dc31aa7cf48b7930a6ce0b220aee3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/7b5dc31aa7cf48b7930a6ce0b220aee3 2024-11-19T17:51:06,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/26afb3745abf4a4093e88ab8adac3f02 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/26afb3745abf4a4093e88ab8adac3f02 2024-11-19T17:51:06,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f578cc44c69e442ca7077102d5ff1d54 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f578cc44c69e442ca7077102d5ff1d54 2024-11-19T17:51:06,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/94fd63105abf4f17b4f228065bad3a4a to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/94fd63105abf4f17b4f228065bad3a4a 2024-11-19T17:51:06,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1efa8376aca347f3aed015795b654076 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1efa8376aca347f3aed015795b654076 2024-11-19T17:51:06,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/98df6ef2d25740b6aa9637fd9744b2d3 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/98df6ef2d25740b6aa9637fd9744b2d3 2024-11-19T17:51:06,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f92871bcd06d43599e692f4df3b44789 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/f92871bcd06d43599e692f4df3b44789 2024-11-19T17:51:06,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/dcc5ce986e6a40669d5a9a4bdf9460b2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/dcc5ce986e6a40669d5a9a4bdf9460b2 2024-11-19T17:51:06,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3f0e0d06122843649bdf6f36af270964 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3f0e0d06122843649bdf6f36af270964 2024-11-19T17:51:06,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/a52fa0f4390b47d08032882e3846b265 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/a52fa0f4390b47d08032882e3846b265 2024-11-19T17:51:06,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/6a3652b1f6e6489b89e419fcc4944fcd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/6a3652b1f6e6489b89e419fcc4944fcd 2024-11-19T17:51:06,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ae9c016c488e4e0cbe44389aa7129957 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ae9c016c488e4e0cbe44389aa7129957 2024-11-19T17:51:06,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2e27ac4146fd4d24828cb3a0b93cee76 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/2e27ac4146fd4d24828cb3a0b93cee76 2024-11-19T17:51:06,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ff2c254ab23d40ec84a4b65ce3b36b22 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/ff2c254ab23d40ec84a4b65ce3b36b22 2024-11-19T17:51:06,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/99482849796b4e6f8bb4241a564dd845 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/99482849796b4e6f8bb4241a564dd845 2024-11-19T17:51:06,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3151a350fdc444f6aaa4a9eed30486e6 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/3151a350fdc444f6aaa4a9eed30486e6 2024-11-19T17:51:06,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/c2b7913b9fc54210aff16e513c0cbf69 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/c2b7913b9fc54210aff16e513c0cbf69 2024-11-19T17:51:06,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/65f6f9918dc047129adb1124376d5b73 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/65f6f9918dc047129adb1124376d5b73 2024-11-19T17:51:06,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/522b0fd5064c4eff91fa56e2b982ed3d to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/522b0fd5064c4eff91fa56e2b982ed3d 2024-11-19T17:51:06,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0f62c8f64a224ed58d9da2c7ac3f7ad7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/0f62c8f64a224ed58d9da2c7ac3f7ad7 2024-11-19T17:51:06,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/287cb4893f3c40018e0794a2a826efcb to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/287cb4893f3c40018e0794a2a826efcb 2024-11-19T17:51:06,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/12309489aa6b476f8e99faae528d798e to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/12309489aa6b476f8e99faae528d798e 2024-11-19T17:51:06,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1b8179ade9d1425090d0b94cc52ae2a7 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/1b8179ade9d1425090d0b94cc52ae2a7 2024-11-19T17:51:06,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/fecf06d871034cd6992ebece031b54f8 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/fecf06d871034cd6992ebece031b54f8 2024-11-19T17:51:06,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d1ba56fc3d9642e2adf49da91bd447fd to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d1ba56fc3d9642e2adf49da91bd447fd 2024-11-19T17:51:06,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d7e369245b7f40b8a01a7d6c1a4184b2 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/d7e369245b7f40b8a01a7d6c1a4184b2 2024-11-19T17:51:06,134 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=4 2024-11-19T17:51:06,134 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. 2024-11-19T17:51:06,135 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for b9c37934b4aa3330f167e8a793f13680: Waiting for close lock at 1732038664401Running coprocessor pre-close hooks at 1732038664401Disabling compacts and flushes for region at 1732038664401Disabling writes for close at 1732038664401Obtaining lock to block concurrent updates at 1732038664402 (+1 ms)Preparing flush snapshotting stores in b9c37934b4aa3330f167e8a793f13680 at 1732038664402Finished memstore snapshotting TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732038664402Flushing stores of TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680. at 1732038664402Flushing b9c37934b4aa3330f167e8a793f13680/A: creating writer at 1732038664402Flushing mob file b9c37934b4aa3330f167e8a793f13680/A: appending metadata at 1732038664407 (+5 ms)Flushing mob file b9c37934b4aa3330f167e8a793f13680/A: closing flushed file at 1732038664407Flushing b9c37934b4aa3330f167e8a793f13680/A: appending metadata at 1732038664815 (+408 ms)Flushing b9c37934b4aa3330f167e8a793f13680/A: closing flushed file at 1732038664815Flushing b9c37934b4aa3330f167e8a793f13680/B: creating writer at 1732038665221 (+406 ms)Flushing b9c37934b4aa3330f167e8a793f13680/B: appending metadata at 1732038665223 (+2 ms)Flushing b9c37934b4aa3330f167e8a793f13680/B: closing flushed file at 1732038665223Flushing b9c37934b4aa3330f167e8a793f13680/C: creating writer at 1732038665632 (+409 ms)Flushing b9c37934b4aa3330f167e8a793f13680/C: appending metadata at 1732038665635 (+3 ms)Flushing b9c37934b4aa3330f167e8a793f13680/C: closing flushed file at 1732038665636 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7752ba17: reopening flushed file at 1732038666050 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78d16015: reopening flushed file at 1732038666053 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@410af06: reopening flushed file at 1732038666056 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for b9c37934b4aa3330f167e8a793f13680 in 1659ms, sequenceid=409, compaction requested=false at 1732038666060 (+4 ms)Writing region close event to WAL at 1732038666131 (+71 ms)Running coprocessor post-close hooks at 1732038666134 (+3 ms)Closed at 1732038666134 2024-11-19T17:51:06,136 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,137 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=b9c37934b4aa3330f167e8a793f13680, regionState=CLOSED 2024-11-19T17:51:06,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 because future has completed 2024-11-19T17:51:06,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-19T17:51:06,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure b9c37934b4aa3330f167e8a793f13680, server=26d3106abcfe,36839,1732038481522 in 1.8900 sec 2024-11-19T17:51:06,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-19T17:51:06,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9c37934b4aa3330f167e8a793f13680, UNASSIGN in 1.8940 sec 2024-11-19T17:51:06,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-19T17:51:06,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8970 sec 2024-11-19T17:51:06,145 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038666145"}]},"ts":"1732038666145"} 2024-11-19T17:51:06,146 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:51:06,147 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:51:06,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9060 sec 2024-11-19T17:51:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:51:06,388 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:51:06,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:51:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:51:06,391 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:51:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:51:06,392 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:51:06,393 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,395 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C, FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/recovered.edits] 2024-11-19T17:51:06,397 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2907c1f36a22496681e5bb17806a62e5 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/2907c1f36a22496681e5bb17806a62e5 2024-11-19T17:51:06,398 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b5b1e652408b46c68b0dd5421793f8ec to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/A/b5b1e652408b46c68b0dd5421793f8ec 2024-11-19T17:51:06,400 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/04f7ae224830441fa7b23d80aebbd577 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/04f7ae224830441fa7b23d80aebbd577 2024-11-19T17:51:06,401 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c8d46161b7bc487487a2efbda75ee10c to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/B/c8d46161b7bc487487a2efbda75ee10c 2024-11-19T17:51:06,402 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/57df53b7040e4b56b224003e7688d647 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/57df53b7040e4b56b224003e7688d647 2024-11-19T17:51:06,403 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/c6bab54ede8743198e2d33b52a40e1ae to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/C/c6bab54ede8743198e2d33b52a40e1ae 2024-11-19T17:51:06,405 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/recovered.edits/412.seqid to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680/recovered.edits/412.seqid 2024-11-19T17:51:06,406 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/default/TestAcidGuarantees/b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,406 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:51:06,406 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:51:06,407 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T17:51:06,409 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111918cf2c06c0e2447890fc8f5000706043_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111918cf2c06c0e2447890fc8f5000706043_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,410 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191bf07175ae5d40adb3923c71a4242620_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191bf07175ae5d40adb3923c71a4242620_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,411 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192002dfd760484753a88f2da4ae0f3bb8_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192002dfd760484753a88f2da4ae0f3bb8_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,412 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111920974a04e5874387b15529a32b538bf0_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111920974a04e5874387b15529a32b538bf0_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,413 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119238c02e3fd9141f885a5e310a9edca33_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119238c02e3fd9141f885a5e310a9edca33_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,414 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193493b929f7374db08055ff282595953f_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193493b929f7374db08055ff282595953f_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,415 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111940d4976b90484f85a47f2a2e4db5b69f_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111940d4976b90484f85a47f2a2e4db5b69f_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,416 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194afef87a9922469dbe8bfd71be3f5b5e_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194afef87a9922469dbe8bfd71be3f5b5e_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,417 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111955573ecf7690404dbd6ce28469fe37fa_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111955573ecf7690404dbd6ce28469fe37fa_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,418 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119816e6a935fdc4b2b958b404afe2e6803_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119816e6a935fdc4b2b958b404afe2e6803_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,419 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198527af70fbf54901bba8b7ba2bc18ef7_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198527af70fbf54901bba8b7ba2bc18ef7_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,420 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119899b83e9ec874d018704165ac039b350_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119899b83e9ec874d018704165ac039b350_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,421 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198e31ae259b8144428081c17c8cfc11a7_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198e31ae259b8144428081c17c8cfc11a7_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,422 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198ef177c203234b4d965488367363d3f0_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198ef177c203234b4d965488367363d3f0_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,422 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994083457f9c54d3f90737d1d3a7c03fb_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994083457f9c54d3f90737d1d3a7c03fb_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,423 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a6f97cdbdd3040b093a1df210e23180b_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a6f97cdbdd3040b093a1df210e23180b_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,424 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a92c5524a0a642ffbb9f1889315750d7_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a92c5524a0a642ffbb9f1889315750d7_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,425 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b392cdae497a4c2ea30a79cdd727f526_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b392cdae497a4c2ea30a79cdd727f526_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,426 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b3aaaf1a078946709a54d27bf704d6fb_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b3aaaf1a078946709a54d27bf704d6fb_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,427 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c23ef4bef4894ac6b4846cbef1736659_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c23ef4bef4894ac6b4846cbef1736659_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,428 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d4b6fa9601a343ccb203d2f0d77598a7_b9c37934b4aa3330f167e8a793f13680 to hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d4b6fa9601a343ccb203d2f0d77598a7_b9c37934b4aa3330f167e8a793f13680 2024-11-19T17:51:06,429 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:51:06,431 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:51:06,434 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:51:06,437 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:51:06,438 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:51:06,438 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:51:06,438 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732038666438"}]},"ts":"9223372036854775807"} 2024-11-19T17:51:06,440 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:51:06,440 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b9c37934b4aa3330f167e8a793f13680, NAME => 'TestAcidGuarantees,,1732038634457.b9c37934b4aa3330f167e8a793f13680.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:51:06,440 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:51:06,441 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732038666440"}]},"ts":"9223372036854775807"} 2024-11-19T17:51:06,443 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:51:06,444 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:51:06,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-19T17:51:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41481 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:51:06,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:51:06,500 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:51:06,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=561 (was 617), ProcessCount=11 (was 12), AvailableMemoryMB=10072 (was 10075) 2024-11-19T17:51:06,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T17:51:06,511 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:51:06,511 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:51:06,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:06,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:06,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:51:06,511 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T17:51:06,511 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1201124139, stopped=false 2024-11-19T17:51:06,512 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=26d3106abcfe,41481,1732038480627 2024-11-19T17:51:06,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:51:06,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:51:06,586 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T17:51:06,586 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:51:06,586 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:51:06,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:06,587 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '26d3106abcfe,36839,1732038481522' ***** 2024-11-19T17:51:06,587 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T17:51:06,587 INFO [regionserver/26d3106abcfe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:51:06,587 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T17:51:06,587 INFO [RS:0;26d3106abcfe:36839 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T17:51:06,587 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T17:51:06,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:51:06,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:51:06,588 INFO [RS:0;26d3106abcfe:36839 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T17:51:06,588 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(959): stopping server 26d3106abcfe,36839,1732038481522 2024-11-19T17:51:06,588 INFO [RS:0;26d3106abcfe:36839 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:51:06,588 INFO [RS:0;26d3106abcfe:36839 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;26d3106abcfe:36839. 2024-11-19T17:51:06,588 DEBUG [RS:0;26d3106abcfe:36839 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:51:06,588 DEBUG [RS:0;26d3106abcfe:36839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:06,588 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:51:06,588 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T17:51:06,588 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:51:06,588 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T17:51:06,588 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T17:51:06,589 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T17:51:06,589 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T17:51:06,589 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T17:51:06,590 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:06,590 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:51:06,590 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:51:06,590 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:51:06,590 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:51:06,590 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:51:06,590 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T17:51:06,613 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/info/e82efb909508484fb57a2cb52b3a028a is 95, key is TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370./info:/1732038519581/DeleteFamily/seqid=0 2024-11-19T17:51:06,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742491_1667 (size=6514) 2024-11-19T17:51:06,790 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:06,990 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:07,017 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/info/e82efb909508484fb57a2cb52b3a028a 2024-11-19T17:51:07,035 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/ns/14b38b962fb94c72a136ac6a07418c50 is 93, key is TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370./ns:/1732038519548/DeleteFamily/seqid=0 2024-11-19T17:51:07,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742492_1668 (size=5906) 2024-11-19T17:51:07,190 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:07,391 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:07,442 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/ns/14b38b962fb94c72a136ac6a07418c50 2024-11-19T17:51:07,460 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/rep_barrier/0399f1b4e5d4407cafd3fda815aec602 is 102, key is TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370./rep_barrier:/1732038519548/DeleteFamily/seqid=0 2024-11-19T17:51:07,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742493_1669 (size=6025) 2024-11-19T17:51:07,578 INFO [regionserver/26d3106abcfe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T17:51:07,578 INFO [regionserver/26d3106abcfe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T17:51:07,591 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:07,791 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:07,863 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/rep_barrier/0399f1b4e5d4407cafd3fda815aec602 2024-11-19T17:51:07,887 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/table/9f07ac1d3c4c4fa0a0e357e99ead9e10 is 96, key is TestAcidGuarantees,,1732038486140.66eb19cac62d93a8797a177e6c688370./table:/1732038519548/DeleteFamily/seqid=0 2024-11-19T17:51:07,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742494_1670 (size=5936) 2024-11-19T17:51:07,991 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:08,191 DEBUG [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:51:08,291 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/table/9f07ac1d3c4c4fa0a0e357e99ead9e10 2024-11-19T17:51:08,294 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/info/e82efb909508484fb57a2cb52b3a028a as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/info/e82efb909508484fb57a2cb52b3a028a 2024-11-19T17:51:08,297 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/info/e82efb909508484fb57a2cb52b3a028a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T17:51:08,298 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/ns/14b38b962fb94c72a136ac6a07418c50 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/ns/14b38b962fb94c72a136ac6a07418c50 2024-11-19T17:51:08,300 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/ns/14b38b962fb94c72a136ac6a07418c50, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T17:51:08,301 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/rep_barrier/0399f1b4e5d4407cafd3fda815aec602 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/rep_barrier/0399f1b4e5d4407cafd3fda815aec602 2024-11-19T17:51:08,303 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/rep_barrier/0399f1b4e5d4407cafd3fda815aec602, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T17:51:08,303 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/.tmp/table/9f07ac1d3c4c4fa0a0e357e99ead9e10 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/table/9f07ac1d3c4c4fa0a0e357e99ead9e10 2024-11-19T17:51:08,305 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/table/9f07ac1d3c4c4fa0a0e357e99ead9e10, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T17:51:08,306 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1716ms, sequenceid=90, compaction requested=false 2024-11-19T17:51:08,309 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T17:51:08,309 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T17:51:08,309 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:51:08,309 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732038666589Running coprocessor pre-close hooks at 1732038666590 (+1 ms)Disabling compacts and flushes for region at 1732038666590Disabling writes for close at 1732038666590Obtaining lock to block concurrent updates at 1732038666590Preparing flush snapshotting stores in 1588230740 at 1732038666590Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732038666590Flushing stores of hbase:meta,,1.1588230740 at 1732038666591 (+1 ms)Flushing 1588230740/info: creating writer at 1732038666591Flushing 1588230740/info: appending metadata at 1732038666612 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732038666612Flushing 1588230740/ns: creating writer at 1732038667020 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732038667035 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732038667035Flushing 1588230740/rep_barrier: creating writer at 1732038667445 (+410 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732038667459 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732038667459Flushing 1588230740/table: creating writer at 1732038667866 (+407 ms)Flushing 1588230740/table: appending metadata at 1732038667887 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732038667887Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49c946a8: reopening flushed file at 1732038668294 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6472332: reopening flushed file at 1732038668297 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39c8eb5: reopening flushed file at 1732038668300 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e9a44ea: reopening flushed file at 1732038668303 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1716ms, sequenceid=90, compaction requested=false at 1732038668306 (+3 ms)Writing region close event to WAL at 1732038668307 (+1 ms)Running coprocessor post-close hooks at 1732038668309 (+2 ms)Closed at 1732038668309 2024-11-19T17:51:08,309 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T17:51:08,392 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(976): stopping server 26d3106abcfe,36839,1732038481522; all regions closed. 2024-11-19T17:51:08,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741834_1010 (size=25344) 2024-11-19T17:51:08,398 DEBUG [RS:0;26d3106abcfe:36839 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/oldWALs 2024-11-19T17:51:08,398 INFO [RS:0;26d3106abcfe:36839 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C36839%2C1732038481522.meta:.meta(num 1732038485706) 2024-11-19T17:51:08,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741832_1008 (size=15264714) 2024-11-19T17:51:08,402 DEBUG [RS:0;26d3106abcfe:36839 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/oldWALs 2024-11-19T17:51:08,402 INFO [RS:0;26d3106abcfe:36839 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C36839%2C1732038481522:(num 1732038484639) 2024-11-19T17:51:08,402 DEBUG [RS:0;26d3106abcfe:36839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:51:08,402 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:51:08,402 INFO [RS:0;26d3106abcfe:36839 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:51:08,402 INFO [RS:0;26d3106abcfe:36839 {}] hbase.ChoreService(370): Chore service for: regionserver/26d3106abcfe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T17:51:08,403 INFO [RS:0;26d3106abcfe:36839 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:51:08,403 INFO [regionserver/26d3106abcfe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:51:08,403 INFO [RS:0;26d3106abcfe:36839 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36839 2024-11-19T17:51:08,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:51:08,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/26d3106abcfe,36839,1732038481522 2024-11-19T17:51:08,426 INFO [RS:0;26d3106abcfe:36839 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:51:08,438 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [26d3106abcfe,36839,1732038481522] 2024-11-19T17:51:08,448 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/26d3106abcfe,36839,1732038481522 already deleted, retry=false 2024-11-19T17:51:08,449 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 26d3106abcfe,36839,1732038481522 expired; onlineServers=0 2024-11-19T17:51:08,449 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '26d3106abcfe,41481,1732038480627' ***** 2024-11-19T17:51:08,449 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T17:51:08,449 INFO [M:0;26d3106abcfe:41481 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:51:08,449 INFO [M:0;26d3106abcfe:41481 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:51:08,449 DEBUG [M:0;26d3106abcfe:41481 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T17:51:08,449 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T17:51:08,449 DEBUG [M:0;26d3106abcfe:41481 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T17:51:08,449 DEBUG [master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038484331 {}] cleaner.HFileCleaner(306): Exit Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038484331,5,FailOnTimeoutGroup] 2024-11-19T17:51:08,449 DEBUG [master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038484331 {}] cleaner.HFileCleaner(306): Exit Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038484331,5,FailOnTimeoutGroup] 2024-11-19T17:51:08,450 INFO [M:0;26d3106abcfe:41481 {}] hbase.ChoreService(370): Chore service for: master/26d3106abcfe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T17:51:08,450 INFO [M:0;26d3106abcfe:41481 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:51:08,450 DEBUG [M:0;26d3106abcfe:41481 {}] master.HMaster(1795): Stopping service threads 2024-11-19T17:51:08,450 INFO [M:0;26d3106abcfe:41481 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T17:51:08,450 INFO [M:0;26d3106abcfe:41481 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T17:51:08,450 ERROR [M:0;26d3106abcfe:41481 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T17:51:08,451 INFO [M:0;26d3106abcfe:41481 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T17:51:08,451 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T17:51:08,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T17:51:08,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:51:08,459 DEBUG [M:0;26d3106abcfe:41481 {}] zookeeper.ZKUtil(347): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T17:51:08,460 WARN [M:0;26d3106abcfe:41481 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T17:51:08,460 INFO [M:0;26d3106abcfe:41481 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/.lastflushedseqids 2024-11-19T17:51:08,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742495_1671 (size=72) 2024-11-19T17:51:08,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:51:08,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36839-0x1015435cb240001, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:51:08,539 INFO [RS:0;26d3106abcfe:36839 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:51:08,539 INFO [RS:0;26d3106abcfe:36839 {}] regionserver.HRegionServer(1031): Exiting; stopping=26d3106abcfe,36839,1732038481522; zookeeper connection closed. 2024-11-19T17:51:08,539 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3dd03a2e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3dd03a2e 2024-11-19T17:51:08,540 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T17:51:08,901 INFO [M:0;26d3106abcfe:41481 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T17:51:08,901 INFO [M:0;26d3106abcfe:41481 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T17:51:08,901 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T17:51:08,901 INFO [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:51:08,901 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:51:08,901 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T17:51:08,901 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:51:08,901 INFO [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=771.46 KB heapSize=949.48 KB 2024-11-19T17:51:08,915 DEBUG [M:0;26d3106abcfe:41481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f00f34e310840bf841554366f3c1a29 is 82, key is hbase:meta,,1/info:regioninfo/1732038485797/Put/seqid=0 2024-11-19T17:51:08,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742496_1672 (size=5672) 2024-11-19T17:51:09,318 INFO [M:0;26d3106abcfe:41481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2199 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f00f34e310840bf841554366f3c1a29 2024-11-19T17:51:09,343 DEBUG [M:0;26d3106abcfe:41481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b884813e132248a684dce0330399f0eb is 2276, key is \x00\x00\x00\x00\x00\x00\x00`/proc:d/1732038583673/Put/seqid=0 2024-11-19T17:51:09,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742497_1673 (size=44892) 2024-11-19T17:51:09,750 INFO [M:0;26d3106abcfe:41481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770.90 KB at sequenceid=2199 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b884813e132248a684dce0330399f0eb 2024-11-19T17:51:09,754 INFO [M:0;26d3106abcfe:41481 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b884813e132248a684dce0330399f0eb 2024-11-19T17:51:09,772 DEBUG [M:0;26d3106abcfe:41481 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd11870eb2b642ea8ee7a418db8a3f96 is 69, key is 26d3106abcfe,36839,1732038481522/rs:state/1732038484406/Put/seqid=0 2024-11-19T17:51:09,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073742498_1674 (size=5156) 2024-11-19T17:51:10,178 INFO [M:0;26d3106abcfe:41481 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2199 (bloomFilter=true), to=hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd11870eb2b642ea8ee7a418db8a3f96 2024-11-19T17:51:10,183 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f00f34e310840bf841554366f3c1a29 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2f00f34e310840bf841554366f3c1a29 2024-11-19T17:51:10,186 INFO [M:0;26d3106abcfe:41481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2f00f34e310840bf841554366f3c1a29, entries=8, sequenceid=2199, filesize=5.5 K 2024-11-19T17:51:10,186 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b884813e132248a684dce0330399f0eb as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b884813e132248a684dce0330399f0eb 2024-11-19T17:51:10,189 INFO [M:0;26d3106abcfe:41481 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b884813e132248a684dce0330399f0eb 2024-11-19T17:51:10,189 INFO [M:0;26d3106abcfe:41481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b884813e132248a684dce0330399f0eb, entries=174, sequenceid=2199, filesize=43.8 K 2024-11-19T17:51:10,190 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fd11870eb2b642ea8ee7a418db8a3f96 as hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fd11870eb2b642ea8ee7a418db8a3f96 2024-11-19T17:51:10,193 INFO [M:0;26d3106abcfe:41481 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44619/user/jenkins/test-data/4d29887f-741d-8fc2-fd5e-f377f10a135d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fd11870eb2b642ea8ee7a418db8a3f96, entries=1, sequenceid=2199, filesize=5.0 K 2024-11-19T17:51:10,193 INFO [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(3140): Finished flush of dataSize ~771.46 KB/789974, heapSize ~949.18 KB/971960, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1292ms, sequenceid=2199, compaction requested=false 2024-11-19T17:51:10,194 INFO [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:51:10,195 DEBUG [M:0;26d3106abcfe:41481 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732038668901Disabling compacts and flushes for region at 1732038668901Disabling writes for close at 1732038668901Obtaining lock to block concurrent updates at 1732038668901Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732038668901Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=789974, getHeapSize=972200, getOffHeapSize=0, getCellsCount=2398 at 1732038668902 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732038668902Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732038668902Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732038668915 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732038668915Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732038669322 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732038669343 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732038669343Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732038669754 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732038669771 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732038669771Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6158411d: reopening flushed file at 1732038670182 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@487eaad2: reopening flushed file at 1732038670186 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2adca707: reopening flushed file at 1732038670189 (+3 ms)Finished flush of dataSize ~771.46 KB/789974, heapSize ~949.18 KB/971960, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1292ms, sequenceid=2199, compaction requested=false at 1732038670193 (+4 ms)Writing region close event to WAL at 1732038670194 (+1 ms)Closed at 1732038670194 2024-11-19T17:51:10,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45951 is added to blk_1073741830_1006 (size=934124) 2024-11-19T17:51:10,198 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:51:10,198 INFO [M:0;26d3106abcfe:41481 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T17:51:10,198 INFO [M:0;26d3106abcfe:41481 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41481 2024-11-19T17:51:10,198 INFO [M:0;26d3106abcfe:41481 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:51:10,412 INFO [M:0;26d3106abcfe:41481 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:51:10,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:51:10,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41481-0x1015435cb240000, quorum=127.0.0.1:51092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:51:10,420 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T17:51:10,423 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T17:51:10,423 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T17:51:10,423 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T17:51:10,424 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/hadoop.log.dir/,STOPPED} 2024-11-19T17:51:10,427 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T17:51:10,427 WARN [BP-1826768862-172.17.0.2-1732038476880 heartbeating to localhost/127.0.0.1:44619 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T17:51:10,427 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T17:51:10,427 WARN [BP-1826768862-172.17.0.2-1732038476880 heartbeating to localhost/127.0.0.1:44619 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1826768862-172.17.0.2-1732038476880 (Datanode Uuid b611bc95-043b-4517-91a5-22a5c778988f) service to localhost/127.0.0.1:44619 2024-11-19T17:51:10,430 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/data/data1/current/BP-1826768862-172.17.0.2-1732038476880 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T17:51:10,430 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/cluster_384ef9bd-8138-429e-6b0c-b164b1295f19/data/data2/current/BP-1826768862-172.17.0.2-1732038476880 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T17:51:10,431 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T17:51:10,440 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T17:51:10,440 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T17:51:10,440 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T17:51:10,441 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T17:51:10,441 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a4c3f5f-3894-9bde-4803-e5831757c3c4/hadoop.log.dir/,STOPPED} 2024-11-19T17:51:10,462 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T17:51:10,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down